test(CI): upload junit reports to datadog

Signed-off-by: Alexandre Rulleau <alexandre.rulleau@datadoghq.com>

Author: Leiyks

.github/CODEOWNERS

@@ -154,6 +154,16 @@
 
       TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
       TERM=dumb ./gradlew saveCaches --info
+  after_script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
+  artifacts:
+    reports:
+      junit: "artifacts/**/test-results/**/TEST-*.xml"
+    paths:
+      - "artifacts/"
+    when: "always"
   cache:
     - key: "appsec int test cache"
       paths:

.gitlab/generate-appsec.php

@@ -46,14 +46,23 @@
     - '# NTS'
     - command -v switch-php && switch-php "${PHP_MAJOR_MINOR}"
     - cargo build --profile profiler-release --all-features
-    - (cd tests; php run-tests.php -d "extension=/mnt/ramdisk/cargo/profiler-release/libdatadog_php_profiling.so" --show-diff -g "FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP" "phpt")
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/profiler-tests"
+    - (cd tests; TEST_PHP_JUNIT="${CI_PROJECT_DIR}/artifacts/profiler-tests/nts-results.xml" php run-tests.php -d "extension=/mnt/ramdisk/cargo/profiler-release/libdatadog_php_profiling.so" --show-diff -g "FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP" "phpt")
 
     - touch build.rs #make sure `build.rs` gets executed after `switch-php` call
 
     - '# ZTS'
     - command -v switch-php && switch-php "${PHP_MAJOR_MINOR}-zts"
     - cargo build --profile profiler-release --all-features
-    - (cd tests; php run-tests.php -d "extension=/mnt/ramdisk/cargo/profiler-release/libdatadog_php_profiling.so" --show-diff -g "FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP" "phpt")
+    - (cd tests; TEST_PHP_JUNIT="${CI_PROJECT_DIR}/artifacts/profiler-tests/zts-results.xml" php run-tests.php -d "extension=/mnt/ramdisk/cargo/profiler-release/libdatadog_php_profiling.so" --show-diff -g "FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP" "phpt")
+  after_script:
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:profiling"
+  artifacts:
+    reports:
+      junit: "artifacts/profiler-tests/*.xml"
+    paths:
+      - "artifacts/"
+    when: "always"
 
 "clippy NTS":
   stage: test

.gitlab/generate-profiler.php

@@ -28,13 +28,19 @@
     - mkdir -p tmp/build_php_components_asan && cd tmp/build_php_components_asan
     - cmake $([ -f "/etc/debian_version" ] && echo "-DCMAKE_TOOLCHAIN_FILE=../../cmake/asan.cmake") -DCMAKE_BUILD_TYPE=Debug -DDATADOG_PHP_TESTING=ON ../../components
     - make -j all
-    - make test
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/components-asan-results.xml --output-on-failure"
   after_script:
     - mkdir -p tmp/artifacts
-    - cp tmp/build_php_components_asan/Testing/Temporary/LastTest.log tmp/artifacts/LastTestUBSan.log
+    - cp tmp/build_php_components_asan/Testing/Temporary/LastTest.log tmp/artifacts/LastTestASan.log
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:components-rs"
   artifacts:
+    reports:
+      junit: "artifacts/*-results.xml"
     paths:
       - tmp/artifacts
+      - artifacts
+    when: "always"
 
 "C components UBSAN":
   tags: [ "arch:amd64" ]
@@ -46,13 +52,19 @@
     - mkdir -p tmp/build_php_components_ubsan && cd tmp/build_php_components_ubsan
     - CMAKE_PREFIX_PATH=/opt/catch2 cmake -DCMAKE_TOOLCHAIN_FILE=../../cmake/ubsan.cmake -DCMAKE_BUILD_TYPE=Debug -DDATADOG_PHP_TESTING=ON ../../components
     - make -j all
-    - make test ARGS="--output-on-failure --repeat until-fail:10" # channel is non-deterministic, so run tests a few more times. At the moment, Catch2 tests are not automatically adding labels, so run all tests instead of just channel's: https://github.com/catchorg/Catch2/issues/1590
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/components-ubsan-results.xml --output-on-failure --repeat until-fail:10" # channel is non-deterministic, so run tests a few more times. At the moment, Catch2 tests are not automatically adding labels, so run all tests instead of just channel's: https://github.com/catchorg/Catch2/issues/1590
   after_script:
     - mkdir -p tmp/artifacts
-    - cp tmp/build_php_components_ubsan/Testing/Temporary/LastTest.log tmp/artifacts/LastTestASan.log
+    - cp tmp/build_php_components_ubsan/Testing/Temporary/LastTest.log tmp/artifacts/LastTestUBSan.log
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:components-rs"
   artifacts:
+    reports:
+      junit: "artifacts/*-results.xml"
     paths:
       - tmp/artifacts
+      - artifacts
+    when: "always"
 
 "Build & Test Tea":
   tags: [ "arch:amd64" ]
@@ -67,15 +79,21 @@
   script:
     - sh .gitlab/build-tea.sh $SWITCH_PHP_VERSION
     - cd tmp/build-tea-${SWITCH_PHP_VERSION}
-    - make test
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/tea-${SWITCH_PHP_VERSION}-results.xml --output-on-failure"
     - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true
   after_script:
     - mkdir -p tmp/artifacts/
-    - cp tmp/build-tea-${SWITCH_PHP_VERSION}/Testing/Temporary/LastTest.log tmp/artifacts/LastTestASan.log
+    - cp tmp/build-tea-${SWITCH_PHP_VERSION}/Testing/Temporary/LastTest.log tmp/artifacts/LastTest.log
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:zend_abstract_interface"
   artifacts:
+    reports:
+      junit: "artifacts/*-results.xml"
     paths:
       - tmp/tea
       - tmp/artifacts
+      - artifacts
+    when: "always"
 
 .tea_test:
   tags: [ "arch:amd64" ]
@@ -89,9 +107,14 @@
   after_script:
     - mkdir -p tmp/artifacts
     - cp tmp/build*/Testing/Temporary/LastTest.log tmp/artifacts/LastTest.log
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:zend_abstract_interface"
   artifacts:
+    reports:
+      junit: "artifacts/*-results.xml"
     paths:
       - tmp/artifacts
+      - artifacts
+    when: "always"
 
 <?php
 foreach ($all_minor_major_targets as $major_minor):
@@ -118,7 +141,8 @@
     - mkdir -p tmp/build_zai && cd tmp/build_zai
     - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/<?= $switch_php_version ?> cmake <?= $toolchain ?> -DCMAKE_BUILD_TYPE=Debug -DBUILD_ZAI_TESTING=ON -DPhpConfig_ROOT=$(php-config --prefix) ../../zend_abstract_interface
     - make -j all
-    - make test
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/zai-<?= $major_minor ?>-<?= $switch_php_version ?>-results.xml --output-on-failure"
     - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true
 <?php
     endforeach;

.gitlab/generate-shared.php

@@ -23,6 +23,7 @@ function after_script($execute_dir = ".", $has_test_agent = false) {
     - .gitlab/check_test_agent.sh
 <?php endif; ?>
     - .gitlab/collect_artifacts.sh "<?= $execute_dir ?>"
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:src"
 <?php
 }
 
@@ -246,7 +247,9 @@ function before_script_steps() {
     MAX_TEST_PARALLELISM: 2
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "<?= $arch ?>"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests"
     - make test_c
 <?php after_script("tmp/build_extension", has_test_agent: true); ?>
 
@@ -262,7 +265,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "<?= $arch ?>"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_internal_api_randomized
 <?php after_script(); ?>
 
@@ -288,7 +293,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_with_init_hook
 <?php after_script(); ?>
 
@@ -310,7 +317,9 @@ function before_script_steps() {
     MAX_TEST_PARALLELISM: 4
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests"
     - make test_c_observer
 <?php after_script("tmp/build_extension", has_test_agent: true); ?>
 <?php endif; ?>
@@ -327,7 +336,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_opcache
 <?php after_script(); ?>
 <?php
@@ -378,7 +389,7 @@ function before_script_steps() {
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
   script:
-    - make test_unit <?= ASSERT_NO_MEMLEAKS ?>
+    - make test_unit PHPUNIT_JUNIT="artifacts/tests/php-tests.xml" <?= ASSERT_NO_MEMLEAKS ?>
 <?php after_script(); ?>
 
 "API unit tests: [<?= $major_minor ?>]":
@@ -393,7 +404,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_api_unit <?= ASSERT_NO_MEMLEAKS ?>
 <?php after_script(); ?>
 
@@ -409,6 +422,7 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
 <?php if (version_compare($major_minor, "7.4", ">=")): ?>
     KUBERNETES_CPU_REQUEST: 8
     MAX_TEST_PARALLELISM: 16
@@ -417,6 +431,7 @@ function before_script_steps() {
   timeout: 40m
 <?php endif; ?>
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_c_disabled <?= ASSERT_NO_MEMLEAKS ?>
 <?php after_script(); ?>
 
@@ -432,7 +447,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
     - make test_internal_api_randomized
 <?php after_script(); ?>
 
@@ -448,7 +465,9 @@ function before_script_steps() {
   variables:
     PHP_MAJOR_MINOR: "<?= $major_minor ?>"
     ARCH: "amd64"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests/php-tests.xml"
   script:
+    - mkdir -p "${CI_PROJECT_DIR}/tmp/build_extension/artifacts/tests"
     - make test_opcache
 <?php after_script("tmp/build_extension"); ?>
 
@@ -471,7 +490,7 @@ function before_script_steps() {
     DD_TRACE_WARN_LEGACY_DD_TRACE: "0"
     DD_TRACE_GIT_METADATA_ENABLED: "0"
     REPORT_EXIT_STATUS: "1"
-    TEST_PHP_JUNIT: "/tmp/artifacts/tests/php-tests.xml"
+    TEST_PHP_JUNIT: "${CI_PROJECT_DIR}/artifacts/tests/php-tests.xml"
     SKIP_ONLINE_TEST: "1"
 <?php if (version_compare($major_minor, "7.2", ">=")): /* too expensive */ ?>
     DD_INSTRUMENTATION_TELEMETRY_ENABLED: 0
@@ -504,7 +523,7 @@ function before_script_steps() {
     - make composer_tests_update
     - .gitlab/wait-for-service-ready.sh
   script:
-    - DD_TRACE_AGENT_TIMEOUT=1000 make $MAKE_TARGET RUST_DEBUG_BUILD=1 PHPUNIT_OPTS="--log-junit artifacts/tests/results.xml" <?= ASSERT_NO_MEMLEAKS ?>
+    - DD_TRACE_AGENT_TIMEOUT=1000 make $MAKE_TARGET RUST_DEBUG_BUILD=1 PHPUNIT_JUNIT="artifacts/tests/results.xml" <?= ASSERT_NO_MEMLEAKS ?>
 <?php after_script(".", true); ?>
     - find tests -type f \( -name 'phpunit_error.log' -o -name 'nginx_*.log' -o -name 'apache_*.log' -o -name 'php_fpm_*.log' -o -name 'dd_php_error.log' \) -exec cp --parents '{}' artifacts \;
     - make tested_versions && cp tests/tested_versions/tested_versions.json artifacts/tested_versions_${MAKE_TARGET}_${PHP_MAJOR_MINOR}_${DD_TRACE_TEST_SAPI:-cli}.json
@@ -685,7 +704,7 @@ function before_script_steps() {
     - php /usr/local/src/php/run-tests.php -g FAIL,XFAIL,BORK,WARN,LEAK,XLEAK,SKIP -p $(which php) --show-all -d zend_extension=xdebug-<?= $xdebug ?>.so "tests/xdebug/<?= $xdebug[0] == 2 ? $xdebug : "3.0.0" ?>"
 <?php if ($xdebug != "2.7.2" && $xdebug != "2.9.2"): ?>
     - '# Run unit tests with xdebug'
-    - TEST_EXTRA_INI='-d zend_extension=xdebug-<?= $xdebug ?>.so' make test_unit RUST_DEBUG_BUILD=1 PHPUNIT_OPTS="--log-junit test-results/php-unit/results_unit.xml"
+    - TEST_EXTRA_INI='-d zend_extension=xdebug-<?= $xdebug ?>.so' make test_unit RUST_DEBUG_BUILD=1 PHPUNIT_JUNIT="test-results/php-unit/results_unit.xml"
 <?php endif; ?>
 <?php after_script(has_test_agent: true); ?>
 

.gitlab/generate-tracer.php

@@ -15,7 +15,7 @@ if [[ ! "${XFAIL_LIST:-none}" == "none" ]]; then
 fi
 
 cd /usr/local/src/php
-mkdir -p /tmp/artifacts/tests
+mkdir -p "${CI_PROJECT_DIR}/artifacts/tests"
 # replace all hardcoded object ids in tests by %d as ddtrace creates its own objects
 php <<'PHP'
 <?php