Merge pull request #1749 from Idclip/weekly_ci

Fixed the weekly CI, introduce a new build type for the ABI check job

Author: Idclip

.github/workflows/weekly.yml

@@ -162,7 +162,8 @@ jobs:
       matrix:
         config:
           - { runson: ubuntu-latest, cxx: g++,     cmake: '' }
-          - { runson: ubuntu-latest, cxx: clang++, cmake: '' }
+          # Disable the clang job for now. See https://github.com/actions/runner-images/issues/8659
+          # - { runson: ubuntu-latest, cxx: clang++, cmake: '' }
           # @todo gcc on macos
           - { runson: macos-latest,  cxx: '',      cmake: '-D CMAKE_CXX_COMPILER=/usr/local/opt/llvm@15/bin/clang++' }
       fail-fast: false
@@ -379,74 +380,69 @@ jobs:
   #############################################################################
 
   linux-abi-checker:
-    # v10.0.0 doesn't exist yet, so can't run this automatically.
     if: |
       github.event_name == 'workflow_dispatch' &&
       (github.event.inputs.type == 'all' ||
        github.event.inputs.type == 'abi')
-    # abi-dumper version verified to work with 20.04/GCC9
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
-      VDB_MAJOR_VERSION: 10
+      # The 'abicheck' build type sets these, but older versions of the library
+      # may not have this build type. See OpenVDBCXX.cmake
+      CXXFLAGS: "-gdwarf-4 -g3 -ggdb -Og"
     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
+        fetch-tags: true
+    # Compute the latest major version - that is used as our baseline
+    # note: For CI forks, make sure you have your tags synced
+    - name: get_major_version
+      run: |
+        LATEST_VERSION_TAG=$(git tag --merged | sort --version-sort | tail -n1)
+        echo "Computed latest VDB tag: ${LATEST_VERSION_TAG}"
+        VDB_MAJOR_VERSION=$(echo ${LATEST_VERSION_TAG} | cut -f1 -d '.' | tr -d -c 0-9)
+        echo "Using major version: ${VDB_MAJOR_VERSION}"
+        echo "VDB_MAJOR_VERSION=${VDB_MAJOR_VERSION}" >> "$GITHUB_ENV"
     - name: install_deps
-      run: sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev elfutils
-    # abi-compliance-checker and abi-dumper
-    #
-    # @note that abi-dumper is available through apt but at the time of writing this
-    # the version there (1.1) doesn't work correctly and maniftest by creating an
-    # invalid ABI report with missing headers. This then always reports 100% success
-    # rate when used with abi-compliance-checker.
-    # To fix, install both from source and checkout specific commits for both
-    # which have been verified to work on ubuntu 20.04.
-    #
-    # @warning  If you update these, test that they fail when expected!
-    #
-    # Also note that these are far superior to abigail/abidiff tools from redhat
+      run: sudo apt-get -q install -y libboost-iostreams-dev libtbb-dev libblosc-dev elfutils
     - name: install_abi_checker
-      run: |
-        git clone https://github.com/lvc/abi-dumper.git abi-dumper
-        cd abi-dumper && git checkout 16bb467cd7d343dd3a16782b151b56cf15509594 && cd -
-        git clone https://github.com/lvc/abi-compliance-checker abi-compliance-checker
-        cd abi-compliance-checker && git checkout 7c175c45a8ba9ac41b8e47d8ebbab557b623b18e && cd -
-    - name: build_latest
+      run: sudo apt-get -q install -y abi-dumper abi-compliance-checker
+    - name: build_new
       run: >
-        sudo ./ci/build.sh -v
-        --build-dir=build_latest
-        --build-type=Debug
+        ./ci/build.sh -v
+        --build-dir=build_new
+        --build-type=abicheck
         --target=openvdb_shared
         --components=\"core\"
         --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON\'
     - name: checkout_baseline
       run: git checkout v${VDB_MAJOR_VERSION}.0.0
-    - name: build_baseline
+    - name: build_old
       run: >
-        sudo ./ci/build.sh -v
-        --build-type=Debug
+        ./ci/build.sh -v
+        --build-dir=build_old
+        --build-type=abicheck
         --target=openvdb_shared
         --components=\"core\"
         --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON\'
     - name: abi_dump
       run: |
-        abi-dumper/abi-dumper.pl build_latest/openvdb/openvdb/libopenvdb.so -o ABI-1.dump -lver 1
-        abi-dumper/abi-dumper.pl        build/openvdb/openvdb/libopenvdb.so -o ABI-2.dump -lver 2
+        abi-dumper build_new/openvdb/openvdb/libopenvdb.so -o ABI-NEW.dump -lver 1
+        abi-dumper build_old/openvdb/openvdb/libopenvdb.so -o ABI-OLD.dump -lver 2
       # Replace the version namespace in the latest ABI dump with the baseline
       # version we're comparing against. We should probably instead build the
       # latest with the baseline version number but no CMake/defines allow us to
       # do this.
     - name: replace_symbols
-      run: sed -i -E 's/openvdb([^v]*)v'${VDB_MAJOR_VERSION}'_[0-9]/openvdb\1v'${VDB_MAJOR_VERSION}'_0/g' ABI-1.dump
+      run: sed -i -E 's/openvdb([^v]*)v[0-9]*_[0-9]/openvdb\1v'${VDB_MAJOR_VERSION}'_0/g' ABI-NEW.dump
     - name: abi_check
       # -strict treats warnings as errors
       # -extended checks all member data
       # we check everything _not_ in openvdb::**::internal namespace
       run: >
-        abi-compliance-checker/abi-compliance-checker.pl -l OPENVDB
-        -old ABI-2.dump
-        -new ABI-1.dump
+        abi-compliance-checker -l OPENVDB
+        -old ABI-OLD.dump
+        -new ABI-NEW.dump
         -skip-internal-symbols "\d(openvdb.*internal)"
         -skip-internal-types "(openvdb.*internal)::"
         -strict

cmake/config/OpenVDBCXX.cmake

@@ -245,7 +245,7 @@ if(CMAKE_BUILD_TYPE EQUAL coverage)
 endif()
 
 # Note that the thread, address and memory sanitizers are incompatible with each other
-set(EXTRA_BUILD_TYPES coverage tsan asan lsan msan ubsan)
+set(EXTRA_BUILD_TYPES coverage tsan asan lsan msan ubsan abicheck)
 
 # Set all build flags to empty (unless they have been provided)
 
@@ -304,6 +304,13 @@ add_link_options("$<$<AND:$<CONFIG:MSAN>,$<COMPILE_LANG_AND_ID:CXX,Clang,AppleCl
 add_compile_options("$<$<AND:$<CONFIG:UBSAN>,$<COMPILE_LANG_AND_ID:CXX,GNU,Clang,AppleClang>>:-fsanitize=undefined>")
 add_link_options("$<$<AND:$<CONFIG:UBSAN>,$<COMPILE_LANG_AND_ID:CXX,GNU,Clang,AppleClang>>:-fsanitize=undefined>")
 
+# ABI Check. This build type is expected to work with the abi-dumper/abi-compliance-checker
+# binaries which expect specific debug information. In particular, for GCC versions >= 11
+# we have to explicitly select dwarf versions < 5 as the abi-dumper doesn't support dwarf5
+# and will always incorrectly report successful ABI checks
+#   https://github.com/lvc/abi-dumper/issues/33
+add_compile_options("$<$<CONFIG:ABICHECK>:-gdwarf-4;-g3;-ggdb;-Og>")
+
 # CMAKE_BUILD_TYPE is ignored for multi config generators i.e. MSVS
 
 get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)

cmake/scripts/lsan.supp

@@ -0,0 +1,15 @@
+#################################################################################
+## This file is loaded by the Leak/Address Sanitizer build for the unit tests. ##
+## It can be used to ignore various errors reported by the sanitizer. This is  ##
+## especially useful with upstream issues (e.g. boost/tbb). For help defining  ##
+## suppression rules, see:                                                     ##
+##   https://clang.llvm.org/docs/AddressSanitizer.html                         ##
+## The build is configured with CMAKE_BUILD_TYPE=asan or lsan                  ##
+#################################################################################
+
+##### Upstream #####
+
+# Leaks from TBB init which occur due to tbb teardown issues
+#   https://github.com/oneapi-src/oneTBB/issues/206
+# Should be fixed in oneTBB. Ignore them for now
+leak:tbb::internal::task_stream<3>::initialize*

openvdb/openvdb/unittest/CMakeLists.txt

@@ -221,15 +221,16 @@ endif()
 target_link_libraries(vdb_test ${OPENVDB_TEST_DEPENDENT_LIBS})
 add_test(NAME vdb_unit_test COMMAND $<TARGET_FILE:vdb_test> -v)
 
-# For the undefined behaviour sanitizer, add the suppression file and
-# additional options
-
+# For the sanitizers, add the suppression files and additional options
 get_filename_component(PATH_TO_PROJECT_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 get_filename_component(PATH_TO_PROJECT_ROOT ${PATH_TO_PROJECT_ROOT} DIRECTORY)
 get_filename_component(PATH_TO_PROJECT_ROOT ${PATH_TO_PROJECT_ROOT} DIRECTORY)
+set(LSAN_SUPRESSION_FILE ${PATH_TO_PROJECT_ROOT}/cmake/scripts/lsan.supp)
 set(UBSAN_SUPRESSION_FILE ${PATH_TO_PROJECT_ROOT}/cmake/scripts/ubsan.supp)
 
-set_tests_properties(vdb_unit_test PROPERTIES
-    ENVIRONMENT
-      "$<$<CONFIG:UBSAN>:UBSAN_OPTIONS=halt_on_error=1 report_error_type=1 suppressions=${UBSAN_SUPRESSION_FILE}>")
+set(UBSAN_OPTS "$<$<CONFIG:UBSAN>:UBSAN_OPTIONS=halt_on_error=1 report_error_type=1 suppressions=${UBSAN_SUPRESSION_FILE}>")
+set(LSAN_OPTS  "$<$<CONFIG:LSAN>:LSAN_OPTIONS=suppressions=${LSAN_SUPRESSION_FILE}>")
+set(ASAN_OPTS  "$<$<CONFIG:ASAN>:LSAN_OPTIONS=suppressions=${LSAN_SUPRESSION_FILE}>")
 
+set_tests_properties(vdb_unit_test PROPERTIES
+    ENVIRONMENT "$<JOIN:${UBSAN_OPTS};${LSAN_OPTS};${ASAN_OPTS}, >")

openvdb_ax/openvdb_ax/test/CMakeLists.txt

@@ -133,14 +133,16 @@ endif()
 
 add_test(NAME vdb_ax_unit_test COMMAND vdb_ax_test -v WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../)
 
-# For the undefined behaviour sanitizer, add the suppression file and
-# additional options
-
+# For the sanitizers, add the suppression files and additional options
 get_filename_component(PATH_TO_PROJECT_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 get_filename_component(PATH_TO_PROJECT_ROOT ${PATH_TO_PROJECT_ROOT} DIRECTORY)
 get_filename_component(PATH_TO_PROJECT_ROOT ${PATH_TO_PROJECT_ROOT} DIRECTORY)
+set(LSAN_SUPRESSION_FILE ${PATH_TO_PROJECT_ROOT}/cmake/scripts/lsan.supp)
 set(UBSAN_SUPRESSION_FILE ${PATH_TO_PROJECT_ROOT}/cmake/scripts/ubsan.supp)
 
+set(UBSAN_OPTS "$<$<CONFIG:UBSAN>:UBSAN_OPTIONS=halt_on_error=1 report_error_type=1 suppressions=${UBSAN_SUPRESSION_FILE}>")
+set(LSAN_OPTS  "$<$<CONFIG:LSAN>:LSAN_OPTIONS=suppressions=${LSAN_SUPRESSION_FILE}>")
+set(ASAN_OPTS  "$<$<CONFIG:ASAN>:LSAN_OPTIONS=suppressions=${LSAN_SUPRESSION_FILE}>")
+
 set_tests_properties(vdb_ax_unit_test PROPERTIES
-    ENVIRONMENT
-      "$<$<CONFIG:UBSAN>:UBSAN_OPTIONS=halt_on_error=1 report_error_type=1 suppressions=${UBSAN_SUPRESSION_FILE}>")
+    ENVIRONMENT "$<JOIN:${UBSAN_OPTS};${LSAN_OPTS};${ASAN_OPTS}, >")