Skip to content

Commit ea5b229

Browse files
IdclipIdclip
authored
Merge pull request #1754 from Idclip/houdini_ci
Introduced CI for Houdini on MacOS
2 parents 58e43d3 + de204cd commit ea5b229

File tree

4 files changed

+205
-53
lines changed

4 files changed

+205
-53
lines changed

‎.github/workflows/houdini.yml‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,7 @@ jobs:
109109
restore-keys: vdb-v5-houdini${{ matrix.config.hou_hash }}-
110110
- name: validate_houdini
111111
run: test -f "hou/hou.tar.gz"
112-
# Make sure the cache is copied, not moved, as the cache action always posts the cache.
113-
# Also make sure that the unpacked install is NOT in the root of the OpenVDB checkout
112+
# Make sure that the unpacked install is NOT in the root of the OpenVDB checkout
114113
# otherwise CMake's install RPATHs wil not work correctly.
115114
- name: install_houdini
116115
run: |
@@ -135,6 +134,48 @@ jobs:
135134
- name: ccache_clean
136135
if: matrix.config.build == 'Release'
137136
run: ccache --evict-older-than 1d
138-
# Delete the houdini tarball so that this dummy cache occupies no space
139-
- name: delete_hou
140-
run: rm -f hou/hou.tar.gz
137+
138+
macos-houdini:
139+
needs: [checksecret]
140+
if: >
141+
${{ needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
142+
github.repository_owner == 'AcademySoftwareFoundation' }}
143+
runs-on: macos-latest
144+
steps:
145+
- uses: actions/checkout@v3
146+
- name: fetch_houdini
147+
uses: actions/cache/restore@v3
148+
with:
149+
path: hou
150+
key: dummy-houdini
151+
restore-keys: vdb-v5-houdini-macos-
152+
- name: validate_houdini
153+
run: test -f "hou/hou.tar.gz"
154+
# Make sure that the unpacked install is NOT in the root of the OpenVDB checkout
155+
# otherwise CMake's install RPATHs wil not work correctly.
156+
- name: install_houdini
157+
run: |
158+
mkdir $HOME/houdini_install
159+
cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
160+
cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
161+
- name: install_deps
162+
run: |
163+
brew update
164+
brew install bash gnu-getopt cmake boost glfw googletest openexr pybind11 llvm@15 cppunit
165+
echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH
166+
- name: build
167+
run: |
168+
./ci/build.sh -v \
169+
--build-type=Release \
170+
--components="core,hou,bin,view,render,python,test,axcore,axbin,axtest" \
171+
--cargs=\" \
172+
-DHOUDINI_ROOT=$HOME/houdini_install/hou \
173+
-DOPENVDB_BUILD_HOUDINI_ABITESTS=OFF \
174+
-DOPENVDB_HOUDINI_INSTALL_PREFIX=/tmp \
175+
-DDISABLE_CMAKE_SEARCH_PATHS=ON \
176+
-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
177+
-DUSE_EXPLICIT_INSTANTIATION=OFF \
178+
-DLLVM_DIR=/usr/local/opt/llvm@15/lib/cmake/llvm \
179+
\"
180+
- name: test
181+
run: cd build && ctest -V

‎.github/workflows/weekly.yml‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
# download Houdini and cache it. The secrets are used in download_houdini.py
3535
checksecret:
3636
name: Verify Houdini Secrets
37-
runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
37+
runs-on: ubuntu-latest
3838
outputs:
3939
HOUDINI_SECRETS: ${{ steps.check.outputs.HOUDINI_SECRETS }}
4040
steps:
@@ -46,18 +46,21 @@ jobs:
4646
- name: Skip Next Jobs
4747
if: steps.check.outputs.HOUDINI_SECRETS != 'true'
4848
run: echo "HOUDINI_CLIENT_ID and HOUDINI_SECRET_KEY GitHub Action Secrets needs to be set to install Houdini builds"
49+
# Explicitly error on the ASWF repo, we expect this secret to always exist
50+
- name: Error ASWF
51+
if: steps.check.outputs.HOUDINI_SECRETS != 'true' && github.repository_owner == 'AcademySoftwareFoundation'
52+
run: exit 1
4953

5054
# download the latest production version of Houdini X, strip out headers,
5155
# libraries and binaries required for building OpenVDB and put it into
5256
# the GitHub Actions cache
5357
linux_houdini:
5458
needs: [checksecret]
5559
if: |
56-
(needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
57-
github.repository_owner == 'AcademySoftwareFoundation') &&
60+
(needs.checksecret.outputs.HOUDINI_SECRETS == 'true') &&
5861
(github.event_name != 'workflow_dispatch' ||
59-
github.event.inputs.type == 'all' ||
60-
github.event.inputs.type == 'houdini')
62+
github.event.inputs.type == 'all' ||
63+
github.event.inputs.type == 'houdini')
6164
runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
6265
name: linux-houdini:${{ matrix.config.hou_hash }}
6366
env:
@@ -101,6 +104,36 @@ jobs:
101104
path: hou
102105
key: vdb-v5-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }}
103106

107+
macos_houdini:
108+
needs: [checksecret]
109+
if: |
110+
(needs.checksecret.outputs.HOUDINI_SECRETS == 'true') &&
111+
(github.event_name != 'workflow_dispatch' ||
112+
github.event.inputs.type == 'all' ||
113+
github.event.inputs.type == 'houdini')
114+
runs-on: macos-latest
115+
name: macos-houdini-20
116+
env:
117+
HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
118+
HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
119+
steps:
120+
- uses: actions/checkout@v3
121+
- name: timestamp
122+
id: timestamp
123+
run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
124+
- name: download_houdini
125+
run: ./ci/download_houdini.sh 20.0 macosx_x86_64_clang12.0_11 --prod
126+
- name: install_houdini
127+
run: |
128+
mkdir $HOME/houdini_install
129+
cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
130+
cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
131+
- name: write_houdini_cache
132+
uses: actions/cache/save@v3
133+
with:
134+
path: hou
135+
key: vdb-v5-houdini-macos-${{ steps.timestamp.outputs.timestamp }}
136+
104137
#############################################################################
105138
########################### Core Library Extras #############################
106139
#############################################################################

‎ci/download_houdini.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,9 +182,20 @@ def __init__(self, http_code, message):
182182

183183
latest_release = service.download.get_daily_build_download(
184184
product='houdini', version=version, platform=platform, build=releases_list[0]['build'])
185+
print(latest_release)
186+
187+
# Can't do this procedurally as latest_release['filename'] can contain
188+
# multiple periods and may have multiple trailing extensions...
189+
extension = ''
190+
if 'linux' in platform: extension = 'tar.gz'
191+
elif 'macos' in platform: extension = 'dmg'
192+
elif 'win64' in platform: extension = 'exe'
193+
assert(extension in latest_release['filename'])
194+
195+
# Download the file and save it as hou.extension
196+
local_filename = 'hou.' + extension
197+
print('Writing to "' + local_filename + '"')
185198

186-
# Download the file as hou.tar.gz
187-
local_filename = 'hou.tar.gz'
188199
response = requests.get(latest_release['download_url'], stream=True)
189200
if response.status_code == 200:
190201
response.raw.decode_content = True

‎ci/download_houdini.sh‎

Lines changed: 108 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,53 +3,120 @@
33
set -ex
44

55
HOUDINI_MAJOR="$1"
6-
GOLD="$2"
6+
PLATFORM="$2"
7+
OTHER_ARGS="$3"
78

89
pip install --user requests
10+
python ci/download_houdini.py $HOUDINI_MAJOR $PLATFORM $OTHER_ARGS
911

10-
python ci/download_houdini.py $HOUDINI_MAJOR $GOLD
11-
12-
# create dir hierarchy
13-
mkdir -p hou/bin
14-
mkdir -p hou/houdini
15-
mkdir -p hou/toolkit
16-
mkdir -p hou/dsolib
17-
18-
# unpack hou.tar.gz and cleanup
19-
tar -xzf hou.tar.gz
20-
rm -rf hou.tar.gz
21-
cd houdini*
22-
tar -xzf houdini.tar.gz
23-
24-
# copy required files into hou dir
25-
cp houdini_setup* ../hou/.
26-
27-
# report library names
28-
ls -al dsolib/
29-
30-
# copy required libraries
31-
cp -r toolkit/cmake ../hou/toolkit/.
32-
cp -r toolkit/include ../hou/toolkit/.
33-
cp -r dsolib/libHoudini* ../hou/dsolib/.
34-
cp -r dsolib/libopenvdb_sesi* ../hou/dsolib/.
35-
cp -r dsolib/libblosc* ../hou/dsolib/.
36-
cp -r dsolib/libhboost* ../hou/dsolib/.
37-
cp -r dsolib/libz* ../hou/dsolib/.
38-
cp -r dsolib/libbz2* ../hou/dsolib/.
39-
cp -r dsolib/libtbb* ../hou/dsolib/.
40-
cp -r dsolib/libjemalloc* ../hou/dsolib/.
41-
cp -r dsolib/liblzma* ../hou/dsolib/.
42-
cp -r dsolib/libIex* ../hou/dsolib/.
43-
cp -r dsolib/libImath* ../hou/dsolib/.
44-
cp -r dsolib/libIlmThread* ../hou/dsolib/.
45-
46-
if [ "$HOUDINI_MAJOR" == "19.0" ]; then
47-
cp -r dsolib/libHalf* ../hou/dsolib/.
48-
cp -r dsolib/libIlmImf* ../hou/dsolib/.
12+
if [[ $PLATFORM =~ "linux" ]]; then
13+
# create dir hierarchy
14+
mkdir -p hou/bin
15+
mkdir -p hou/houdini
16+
mkdir -p hou/toolkit
17+
mkdir -p hou/dsolib
18+
19+
# unpack hou.tar.gz and cleanup
20+
tar -xzf hou.tar.gz
21+
rm -rf hou.tar.gz
22+
cd houdini*
23+
tar -xzf houdini.tar.gz
24+
25+
# copy required files into hou dir
26+
cp houdini_setup* ../hou/.
27+
28+
# report library names
29+
ls -al dsolib/
30+
31+
# copy required libraries
32+
cp -r toolkit/cmake ../hou/toolkit/.
33+
cp -r toolkit/include ../hou/toolkit/.
34+
cp -r dsolib/libHoudini* ../hou/dsolib/.
35+
cp -r dsolib/libopenvdb_sesi* ../hou/dsolib/.
36+
cp -r dsolib/libblosc* ../hou/dsolib/.
37+
cp -r dsolib/libhboost* ../hou/dsolib/.
38+
cp -r dsolib/libz* ../hou/dsolib/.
39+
cp -r dsolib/libbz2* ../hou/dsolib/.
40+
cp -r dsolib/libtbb* ../hou/dsolib/.
41+
cp -r dsolib/libjemalloc* ../hou/dsolib/.
42+
cp -r dsolib/liblzma* ../hou/dsolib/.
43+
cp -r dsolib/libIex* ../hou/dsolib/.
44+
cp -r dsolib/libImath* ../hou/dsolib/.
45+
cp -r dsolib/libIlmThread* ../hou/dsolib/.
46+
cd ..
47+
48+
elif [[ $PLATFORM =~ "macos" ]]; then
49+
# Exract files by mounting the downloaded dmg (we only really want to
50+
# expand Houdini.framework)
51+
hdiutil attach hou.dmg
52+
pkgutil --expand-full /Volumes/Houdini/Houdini.pkg Houdini
53+
hdiutil detach /Volumes/Houdini
54+
rm hou.dmg
55+
56+
# Move the required Frameworks and delete the extracted src
57+
mkdir -p hou/Frameworks
58+
mv Houdini/Framework.pkg/Payload/Houdini.framework hou/Frameworks/Houdini.framework
59+
rm -rf Houdini
60+
61+
# Report library names
62+
ls -al hou/Frameworks/Houdini.framework/Libraries
63+
64+
# Remove unused resources
65+
cd hou/Frameworks/Houdini.framework/Resources/
66+
rm -rf $(ls | grep -e toolkit -v)
67+
cd -
68+
69+
# Handle libraries. On some versions of MacOS with older versions of ld,
70+
# ld will complain (error) if shared libraries contain missing files which
71+
# are referenced with LC_LOAD_DYLIB or LC_RPATH entries (even though they
72+
# are not explicitly required at link time). We still want to delete these
73+
# unused libs as they occupt ~1-2GB. To handle this, we generate a unique
74+
# list of libs that our direct dependencies reference and create an empty
75+
# shared dylib in their place.
76+
cd hou/Frameworks/Houdini.framework/Libraries
77+
# Remove any folders here, they aren't needed
78+
rm -rf $(ls -p | grep /)
79+
# Remove any library that does not match the -e patterns (inverse grep with -v)
80+
unused_libraries=$(ls | \
81+
grep -e libHoudini \
82+
-e libopenvdb_sesi \
83+
-e libblosc \
84+
-e libhboost \
85+
-e libz \
86+
-e libbz2 \
87+
-e libtbb \
88+
-e libjemalloc \
89+
-e liblzma \
90+
-e libIex \
91+
-e libImath \
92+
-e libIlmThread \
93+
-v)
94+
rm -rf ${unused_libraries}
95+
96+
# Create an empty valid shared lib
97+
echo '' | clang -x c -shared -o libempty.dylib -
98+
99+
# Generate a unique list of libs that our remaining libs reference
100+
for i in $(ls); do otool -LX $i >> libnames; done
101+
sort -u libnames | grep @rpath | cut -f1 -d' ' | xargs > rpaths
102+
103+
# Recreate unused libraries that have been deleted as empty shared dylibs
104+
# to keep ld happy
105+
for libpath in $(cat rpaths); do
106+
libpath=${libpath#"@rpath/"}
107+
echo "Checking $libpath"
108+
if [ ! -f $libpath ]; then
109+
echo "Creating empty library at $libpath"
110+
mkdir -p $(dirname $libpath)
111+
cp libempty.dylib $libpath
112+
fi
113+
done
114+
115+
rm libempty.dylib
116+
cd -
49117
fi
50118

51119
# write hou into hou.tar.gz and cleanup
52-
cd ..
53120
tar -czvf hou.tar.gz hou
54121

55122
# move hou.tar.gz into hou subdirectory

0 commit comments

Comments
 (0)