From 41acc107fe7bad32d041346d712d994867f740cf Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Wed, 21 Jan 2026 21:22:20 +0100 Subject: [PATCH 1/7] feat: Add DataProvider support to Allure report setup section This commit implements IDataProviderListener in AllureTestNg to report DataProvider executions as fixtures in the Allure report. - Implemented beforeDataProviderExecution, afterDataProviderExecution, and onDataProviderFailure. - DataProvider executions now appear in the 'Set up' section. - Attachments added within DataProviders are now visible. - Failed DataProviders are correctly reported with BROKEN status. - Added tests to verify correct reporting of successful and failed DataProviders. --- .../io/qameta/allure/testng/AllureTestNg.java | 57 ++++++++++++++++++- .../allure/testng/AllureTestNgTest.java | 30 ++++++++++ .../samples/DataProviderWithAttachment.java | 20 +++++++ .../testng/samples/FailedDataProvider.java | 16 ++++++ .../suites/data-provider-with-attachment.xml | 8 +++ .../resources/suites/failed-data-provider.xml | 8 +++ 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java create mode 100644 allure-testng/src/test/resources/suites/data-provider-with-attachment.xml create mode 100644 allure-testng/src/test/resources/suites/failed-data-provider.xml diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java index 41b7c362c..7657f7b8a 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java @@ -39,6 +39,8 @@ import org.testng.IAttributes; import org.testng.IClass; import org.testng.IConfigurationListener; +import org.testng.IDataProviderListener; +import org.testng.IDataProviderMethod; import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; import org.testng.IMethodInstance; @@ -110,7 +112,8 @@ public class AllureTestNg implements ITestListener, IInvokedMethodListener, IConfigurationListener, - IMethodInterceptor { + IMethodInterceptor, + IDataProviderListener { private static final Logger LOGGER = LoggerFactory.getLogger(AllureTestNg.class); @@ -626,6 +629,58 @@ public void onConfigurationSkip(final ITestResult itr) { //do nothing } + @Override + public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMethod, + final ITestNGMethod method, + final ITestContext iTestContext) { + final ITestClass testClass = method.getTestClass(); + final String uuid = currentExecutable.get(); + final FixtureResult result = new FixtureResult() + .setName(dataProviderMethod.getMethod().getName()) + .setStage(Stage.RUNNING); + + processDescription( + getClass().getClassLoader(), + dataProviderMethod.getMethod(), + result::setDescription, + result::setDescriptionHtml + ); + + getClassContainer(testClass).ifPresent(parentUuid -> + getLifecycle().startPrepareFixture(parentUuid, uuid, result) + ); + } + + @Override + public void afterDataProviderExecution(final IDataProviderMethod dataProviderMethod, + final ITestNGMethod method, + final ITestContext iTestContext) { + final ITestClass testClass = method.getTestClass(); + getClassContainer(testClass).ifPresent(parentUuid -> { + final String uuid = currentExecutable.get(); + getLifecycle().updateFixture(uuid, result -> { + if (result.getStatus() == null) { + result.setStatus(Status.PASSED); + } + }); + getLifecycle().stopFixture(uuid); + currentExecutable.remove(); + }); + } + + @Override + public void onDataProviderFailure(final ITestNGMethod method, + final ITestContext ctx, + final RuntimeException t) { + final ITestClass testClass = method.getTestClass(); + getClassContainer(testClass).ifPresent(parentUuid -> { + final String uuid = currentExecutable.get(); + getLifecycle().updateFixture(uuid, result -> result + .setStatus(getStatus(t)) + .setStatusDetails(getStatusDetails(t).orElse(null))); + }); + } + protected String getHistoryId(final ITestNGMethod method, final List parameters) { final MessageDigest digest = getMd5Digest(); final String testClassName = method.getTestClass().getName(); diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index 1276a2120..7d2c41c84 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -1515,6 +1515,36 @@ public AllureResults runTestPlan(final TestPlan plan, final Class... testClas }); } + @AllureFeatures.Fixtures + @Test(description = "Should process data provider in setup") + public void shouldProcessDataProviderInSetup() { + final AllureResults results = runTestNgSuites("suites/data-provider-with-attachment.xml"); + + assertThat(results.getTestResultContainers()) + .flatExtracting(TestResultContainer::getBefores) + .extracting(FixtureResult::getName, FixtureResult::getStatus) + .contains(Tuple.tuple("dataProvider", Status.PASSED)); + + assertThat(results.getTestResultContainers()) + .flatExtracting(TestResultContainer::getBefores) + .filteredOn("name", "dataProvider") + .flatExtracting(FixtureResult::getAttachments) + .hasSize(1) + .extracting(Attachment::getName) + .contains("attachment"); + } + + @AllureFeatures.Fixtures + @Test(description = "Should process failed data provider in setup") + public void shouldProcessFailedDataProviderInSetup() { + final AllureResults results = runTestNgSuites("suites/failed-data-provider.xml"); + + assertThat(results.getTestResultContainers()) + .flatExtracting(TestResultContainer::getBefores) + .extracting(FixtureResult::getName, FixtureResult::getStatus) + .contains(Tuple.tuple("dataProvider", Status.BROKEN)); + } + private Integer getOrderParameter(final TestResult result) { return result.getParameters().stream() .filter(p -> p.getName().equals("order")) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java new file mode 100644 index 000000000..2a08d2bd8 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java @@ -0,0 +1,20 @@ +package io.qameta.allure.testng.samples; + +import io.qameta.allure.Allure; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderWithAttachment { + + @DataProvider + public Object[][] dataProvider() { + Allure.addAttachment("attachment", "attachment content"); + return new Object[][]{ + {"a"} + }; + } + + @Test(dataProvider = "dataProvider") + public void test(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java new file mode 100644 index 000000000..5ca316dc6 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java @@ -0,0 +1,16 @@ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class FailedDataProvider { + + @DataProvider + public Object[][] dataProvider() { + throw new RuntimeException("Data provider failed"); + } + + @Test(dataProvider = "dataProvider") + public void test(String s) { + } +} diff --git a/allure-testng/src/test/resources/suites/data-provider-with-attachment.xml b/allure-testng/src/test/resources/suites/data-provider-with-attachment.xml new file mode 100644 index 000000000..7f9b02c06 --- /dev/null +++ b/allure-testng/src/test/resources/suites/data-provider-with-attachment.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/allure-testng/src/test/resources/suites/failed-data-provider.xml b/allure-testng/src/test/resources/suites/failed-data-provider.xml new file mode 100644 index 000000000..f21c45910 --- /dev/null +++ b/allure-testng/src/test/resources/suites/failed-data-provider.xml @@ -0,0 +1,8 @@ + + + + + + + + From b5eee09a16db606cac3ebc35fe5ffecf905e540a Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Wed, 21 Jan 2026 21:42:34 +0100 Subject: [PATCH 2/7] fix: Isolate DataProvider fixtures to specific test methods Previously, DataProvider fixtures were attached to the Class container, causing them to appear in the setup of all tests within that class. This commit creates a dedicated container for each DataProvider execution and links it only to the specific test method requesting the data. This ensures that DataProvider execution details (and failures) are correctly associated only with the relevant tests. --- .../io/qameta/allure/testng/AllureTestNg.java | 63 ++++++++++++------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java index 7657f7b8a..936560dd6 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java @@ -153,6 +153,10 @@ public class AllureTestNg implements * Store uuid for class test containers. */ private final Map classContainerUuidStorage = new ConcurrentHashMap<>(); + /** + * Store uuid for data provider containers. + */ + private final Map dataProviderContainerUuidStorage = new ConcurrentHashMap<>(); private final ReadWriteLock lock = new ReentrantReadWriteLock(); private final AllureLifecycle lifecycle; private final AllureTestNgTestFilter testFilter; @@ -283,6 +287,14 @@ public void onAfterClass(final ITestClass testClass) { getLifecycle().stopTestContainer(uuid); getLifecycle().writeTestContainer(uuid); }); + dataProviderContainerUuidStorage.entrySet().removeIf(entry -> { + if (entry.getKey().getTestClass().equals(testClass)) { + getLifecycle().stopTestContainer(entry.getValue()); + getLifecycle().writeTestContainer(entry.getValue()); + return true; + } + return false; + }); } @Override @@ -305,6 +317,10 @@ public void onTestStart(final ITestResult testResult) { .map(ITestResult::getMethod) .map(ITestNGMethod::getTestClass) .ifPresent(clazz -> addClassContainerChild(clazz, uuid)); + + Optional.of(testResult) + .map(ITestResult::getMethod) + .ifPresent(method -> addDataProviderContainerChild(method, uuid)); } @SuppressWarnings("BooleanExpressionComplexity") @@ -633,7 +649,14 @@ public void onConfigurationSkip(final ITestResult itr) { public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMethod, final ITestNGMethod method, final ITestContext iTestContext) { - final ITestClass testClass = method.getTestClass(); + final String containerUuid = UUID.randomUUID().toString(); + getLifecycle().startTestContainer( + new TestResultContainer() + .setUuid(containerUuid) + .setName(method.getMethodName()) + ); + dataProviderContainerUuidStorage.put(method, containerUuid); + final String uuid = currentExecutable.get(); final FixtureResult result = new FixtureResult() .setName(dataProviderMethod.getMethod().getName()) @@ -646,39 +669,33 @@ public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMe result::setDescriptionHtml ); - getClassContainer(testClass).ifPresent(parentUuid -> - getLifecycle().startPrepareFixture(parentUuid, uuid, result) - ); + getLifecycle().startPrepareFixture(containerUuid, uuid, result); } @Override public void afterDataProviderExecution(final IDataProviderMethod dataProviderMethod, final ITestNGMethod method, final ITestContext iTestContext) { - final ITestClass testClass = method.getTestClass(); - getClassContainer(testClass).ifPresent(parentUuid -> { - final String uuid = currentExecutable.get(); - getLifecycle().updateFixture(uuid, result -> { - if (result.getStatus() == null) { - result.setStatus(Status.PASSED); - } - }); - getLifecycle().stopFixture(uuid); - currentExecutable.remove(); + final String uuid = currentExecutable.get(); + getLifecycle().updateFixture(uuid, result -> { + if (result.getStatus() == null) { + result.setStatus(Status.PASSED); + } }); + getLifecycle().stopFixture(uuid); + currentExecutable.remove(); } @Override public void onDataProviderFailure(final ITestNGMethod method, final ITestContext ctx, final RuntimeException t) { - final ITestClass testClass = method.getTestClass(); - getClassContainer(testClass).ifPresent(parentUuid -> { - final String uuid = currentExecutable.get(); - getLifecycle().updateFixture(uuid, result -> result - .setStatus(getStatus(t)) - .setStatusDetails(getStatusDetails(t).orElse(null))); - }); + final String uuid = currentExecutable.get(); + getLifecycle().updateFixture(uuid, result -> result + .setStatus(getStatus(t)) + .setStatusDetails(getStatusDetails(t).orElse(null))); + getLifecycle().stopFixture(uuid); + currentExecutable.remove(); } protected String getHistoryId(final ITestNGMethod method, final List parameters) { @@ -882,6 +899,10 @@ private Consumer setStatus(final Status status, final StatusDetails }; } + private void addDataProviderContainerChild(final ITestNGMethod method, final String childUuid) { + this.addChildToContainer(dataProviderContainerUuidStorage.get(method), childUuid); + } + private void addClassContainerChild(final ITestClass clazz, final String childUuid) { this.addChildToContainer(classContainerUuidStorage.get(clazz), childUuid); } From 02dbd0a6233f77eca93597336fb0f75c92114436 Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Wed, 21 Jan 2026 21:52:11 +0100 Subject: [PATCH 3/7] fix: Reuse DataProvider container for retries and handle suppressed failures - Changed dataProviderContainerUuidStorage to use computeIfAbsent to reuse the existing container for a test method if it already exists (e.g. during retry). - Added currentExecutable.remove() in beforeDataProviderExecution to force generation of a new UUID for each execution attempt, ensuring retries are captured as distinct fixtures even if TestNG suppresses failure events or we missed cleanup. - Added FlakyDataProvider sample and test case to verify retry reporting. --- .../io/qameta/allure/testng/AllureTestNg.java | 18 ++++++--- .../allure/testng/AllureTestNgTest.java | 14 +++++++ .../testng/samples/FlakyDataProvider.java | 39 +++++++++++++++++++ .../resources/suites/flaky-data-provider.xml | 8 ++++ 4 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java create mode 100644 allure-testng/src/test/resources/suites/flaky-data-provider.xml diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java index 936560dd6..de75dbd59 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java @@ -649,13 +649,19 @@ public void onConfigurationSkip(final ITestResult itr) { public void beforeDataProviderExecution(final IDataProviderMethod dataProviderMethod, final ITestNGMethod method, final ITestContext iTestContext) { - final String containerUuid = UUID.randomUUID().toString(); - getLifecycle().startTestContainer( - new TestResultContainer() - .setUuid(containerUuid) - .setName(method.getMethodName()) + currentExecutable.remove(); + final String containerUuid = dataProviderContainerUuidStorage.computeIfAbsent( + method, + key -> { + final String uuid = UUID.randomUUID().toString(); + getLifecycle().startTestContainer( + new TestResultContainer() + .setUuid(uuid) + .setName(method.getMethodName()) + ); + return uuid; + } ); - dataProviderContainerUuidStorage.put(method, containerUuid); final String uuid = currentExecutable.get(); final FixtureResult result = new FixtureResult() diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index 7d2c41c84..eff5fac7b 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -1545,6 +1545,20 @@ public void shouldProcessFailedDataProviderInSetup() { .contains(Tuple.tuple("dataProvider", Status.BROKEN)); } + @AllureFeatures.Fixtures + @Test(description = "Should process flaky data provider in setup") + public void shouldProcessFlakyDataProvider() { + final AllureResults results = runTestNgSuites("suites/flaky-data-provider.xml"); + + assertThat(results.getTestResultContainers()) + .flatExtracting(TestResultContainer::getBefores) + .extracting(FixtureResult::getName, FixtureResult::getStatus) + .containsSubsequence( + Tuple.tuple("provide", Status.BROKEN), + Tuple.tuple("provide", Status.PASSED) + ); + } + private Integer getOrderParameter(final TestResult result) { return result.getParameters().stream() .filter(p -> p.getName().equals("order")) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java new file mode 100644 index 000000000..58dd08f30 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java @@ -0,0 +1,39 @@ +package io.qameta.allure.testng.samples; + +import org.testng.IDataProviderMethod; +import org.testng.IRetryDataProvider; +import org.testng.ITestResult; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +public class FlakyDataProvider { + + private static final AtomicInteger COUNTER = new AtomicInteger(0); + + @BeforeClass + public void reset() { + COUNTER.set(0); + } + + public static class Retry implements IRetryDataProvider { + @Override + public boolean retry(IDataProviderMethod method) { + return COUNTER.incrementAndGet() < 2; + } + } + + @DataProvider(retryUsing = Retry.class) + public Object[][] provide() { + if (COUNTER.get() == 0) { + throw new RuntimeException("Simulated DataProvider Failure"); + } + return new Object[][]{{"data"}}; + } + + @Test(dataProvider = "provide") + public void test(String s) { + } +} diff --git a/allure-testng/src/test/resources/suites/flaky-data-provider.xml b/allure-testng/src/test/resources/suites/flaky-data-provider.xml new file mode 100644 index 000000000..6fa5c6e82 --- /dev/null +++ b/allure-testng/src/test/resources/suites/flaky-data-provider.xml @@ -0,0 +1,8 @@ + + + + + + + + From 54416e6187abe3754f2ba377c7c013ce4fc8228a Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Wed, 21 Jan 2026 22:00:36 +0100 Subject: [PATCH 4/7] test: Update container count assertions in AllureTestNgTest Updated container count expectations in multipleSuites and parallelMethods tests to account for the new DataProvider execution containers introduced by the fix. --- .../test/java/io/qameta/allure/testng/AllureTestNgTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index eff5fac7b..8ce0fc941 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -507,7 +507,7 @@ public void multipleSuites() { .hasSize(3); List uids = testResults.stream().map(TestResult::getUuid).collect(Collectors.toList()); assertThat(testContainers).as("Unexpected quantity of testng containers has been written") - .hasSize(8).extracting(TestResultContainer::getName) + .hasSize(9).extracting(TestResultContainer::getName) .contains(beforeMethodName, beforeMethodName, firstTagName, firstSuiteName, secondTagName, secondSuiteName); @@ -553,7 +553,7 @@ public void parallelMethods() { assertThat(testResults).as("Unexpected quantity of testng case results has been written") .hasSize(2001); assertThat(testContainers).as("Unexpected quantity of testng containers has been written") - .hasSize(6006); + .hasSize(6007); assertContainersPerMethod(before1, testContainers, uids); assertContainersPerMethod(before2, testContainers, uids); From 8971c91051d95b778815837b54eef227a02a841b Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Wed, 21 Jan 2026 22:27:45 +0100 Subject: [PATCH 5/7] chore: Apply spotless --- .../samples/DataProviderWithAttachment.java | 15 +++++++++++++++ .../testng/samples/FailedDataProvider.java | 15 +++++++++++++++ .../allure/testng/samples/FlakyDataProvider.java | 16 +++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java index 2a08d2bd8..511681a08 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderWithAttachment.java @@ -1,3 +1,18 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.qameta.allure.testng.samples; import io.qameta.allure.Allure; diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java index 5ca316dc6..d5fdd60ec 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FailedDataProvider.java @@ -1,3 +1,18 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.qameta.allure.testng.samples; import org.testng.annotations.DataProvider; diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java index 58dd08f30..cea3437f3 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/FlakyDataProvider.java @@ -1,8 +1,22 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.qameta.allure.testng.samples; import org.testng.IDataProviderMethod; import org.testng.IRetryDataProvider; -import org.testng.ITestResult; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; From 6c23f0a086f9abc94da8caf4e98b017e7f57e3f2 Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Tue, 27 Jan 2026 21:41:53 +0100 Subject: [PATCH 6/7] test: add more tests for DataProvider support --- .../allure/testng/AllureTestNgTest.java | 83 +++++++++++++++++++ .../samples/DataProviderInheritanceBase.java | 31 +++++++ .../samples/DataProviderInheritanceChild.java | 25 ++++++ .../samples/DataProviderMultipleClasses1.java | 31 +++++++ .../samples/DataProviderMultipleClasses2.java | 31 +++++++ .../samples/DataProviderMultipleTests.java | 35 ++++++++ .../testng/samples/DataProviderParallel.java | 31 +++++++ .../suites/data-provider-inheritance.xml | 8 ++ .../suites/data-provider-multiple-classes.xml | 9 ++ .../suites/data-provider-multiple-tests.xml | 8 ++ .../suites/data-provider-parallel.xml | 8 ++ 11 files changed, 300 insertions(+) create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceBase.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceChild.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses1.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses2.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleTests.java create mode 100644 allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java create mode 100644 allure-testng/src/test/resources/suites/data-provider-inheritance.xml create mode 100644 allure-testng/src/test/resources/suites/data-provider-multiple-classes.xml create mode 100644 allure-testng/src/test/resources/suites/data-provider-multiple-tests.xml create mode 100644 allure-testng/src/test/resources/suites/data-provider-parallel.xml diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index 8ce0fc941..8201ff595 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -1559,6 +1559,89 @@ public void shouldProcessFlakyDataProvider() { ); } + @AllureFeatures.Fixtures + @Test(description = "Should properly link data provider container to test result") + public void shouldProperlyLinkDataProviderContainerToTestResult() { + final AllureResults results = runTestNgSuites("suites/data-provider-with-attachment.xml"); + + final TestResult tr = findTestResultByName(results, "test"); + final TestResultContainer dpContainer = results.getTestResultContainers().stream() + .filter(c -> c.getName().equals("test")) + .findFirst() + .orElseThrow(() -> new IllegalStateException("DP container not found")); + + assertThat(dpContainer.getChildren()) + .contains(tr.getUuid()); + } + + @AllureFeatures.Fixtures + @Test(description = "Should link multiple tests to data provider container") + public void shouldLinkMultipleTestsToDataProviderContainer() { + final AllureResults results = runTestNgSuites("suites/data-provider-multiple-tests.xml"); + + final List test1Results = results.getTestResults().stream() + .filter(tr -> tr.getName().equals("test1")) + .collect(Collectors.toList()); + final List test2Results = results.getTestResults().stream() + .filter(tr -> tr.getName().equals("test2")) + .collect(Collectors.toList()); + + assertThat(test1Results).hasSize(2); + assertThat(test2Results).hasSize(2); + + final TestResultContainer dpContainer1 = findTestContainerByName(results, "test1"); + final TestResultContainer dpContainer2 = findTestContainerByName(results, "test2"); + + assertThat(dpContainer1.getChildren()) + .containsAll(test1Results.stream().map(TestResult::getUuid).collect(Collectors.toList())); + assertThat(dpContainer2.getChildren()) + .containsAll(test2Results.stream().map(TestResult::getUuid).collect(Collectors.toList())); + } + + @AllureFeatures.Fixtures + @Test(description = "Should link inherited data provider") + public void shouldLinkInheritedDataProvider() { + final AllureResults results = runTestNgSuites("suites/data-provider-inheritance.xml"); + + final TestResult testBase = findTestResultByName(results, "testBase"); + final TestResult testChild = findTestResultByName(results, "testChild"); + + final TestResultContainer dpContainerBase = findTestContainerByName(results, "testBase"); + final TestResultContainer dpContainerChild = findTestContainerByName(results, "testChild"); + + assertThat(dpContainerBase.getChildren()).contains(testBase.getUuid()); + assertThat(dpContainerChild.getChildren()).contains(testChild.getUuid()); + } + + @AllureFeatures.Fixtures + @Test(description = "Should link correct data provider in multiple classes") + public void shouldLinkCorrectDataProviderInMultipleClasses() { + final AllureResults results = runTestNgSuites("suites/data-provider-multiple-classes.xml"); + + final TestResult test1 = findTestResultByName(results, "test1"); + final TestResult test2 = findTestResultByName(results, "test2"); + + final TestResultContainer dpContainer1 = findTestContainerByName(results, "test1"); + final TestResultContainer dpContainer2 = findTestContainerByName(results, "test2"); + + assertThat(dpContainer1.getChildren()) + .contains(test1.getUuid()) + .doesNotContain(test2.getUuid()); + assertThat(dpContainer2.getChildren()) + .contains(test2.getUuid()) + .doesNotContain(test1.getUuid()); + } + + @AllureFeatures.Fixtures + @Test(description = "Should process parallel data provider") + public void shouldProcessParallelDataProvider() { + final AllureResults results = runTestNgSuites("suites/data-provider-parallel.xml"); + + assertThat(results.getTestResults()).hasSize(4); + final TestResultContainer dpContainer = findTestContainerByName(results, "test"); + assertThat(dpContainer.getChildren()).hasSize(4); + } + private Integer getOrderParameter(final TestResult result) { return result.getParameters().stream() .filter(p -> p.getName().equals("order")) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceBase.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceBase.java new file mode 100644 index 000000000..eb634c03d --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceBase.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderInheritanceBase { + + @DataProvider + public Object[][] dp() { + return new Object[][]{{"data"}}; + } + + @Test(dataProvider = "dp") + public void testBase(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceChild.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceChild.java new file mode 100644 index 000000000..80ae7cae0 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderInheritanceChild.java @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.Test; + +public class DataProviderInheritanceChild extends DataProviderInheritanceBase { + + @Test(dataProvider = "dp") + public void testChild(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses1.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses1.java new file mode 100644 index 000000000..c2f6f39a5 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses1.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderMultipleClasses1 { + + @DataProvider + public Object[][] dp1() { + return new Object[][]{{"data1"}}; + } + + @Test(dataProvider = "dp1") + public void test1(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses2.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses2.java new file mode 100644 index 000000000..f71e11e90 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleClasses2.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderMultipleClasses2 { + + @DataProvider + public Object[][] dp2() { + return new Object[][]{{"data2"}}; + } + + @Test(dataProvider = "dp2") + public void test2(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleTests.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleTests.java new file mode 100644 index 000000000..160e1d10d --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderMultipleTests.java @@ -0,0 +1,35 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderMultipleTests { + + @DataProvider + public Object[][] dp() { + return new Object[][]{{"data1"}, {"data2"}}; + } + + @Test(dataProvider = "dp") + public void test1(String s) { + } + + @Test(dataProvider = "dp") + public void test2(String s) { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java new file mode 100644 index 000000000..3f7c1bb67 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2024 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DataProviderParallel { + + @DataProvider(parallel = true) + public Object[][] dp() { + return new Object[][]{{"data1"}, {"data2"}, {"data3"}, {"data4"}}; + } + + @Test(dataProvider = "dp") + public void test(String s) { + } +} diff --git a/allure-testng/src/test/resources/suites/data-provider-inheritance.xml b/allure-testng/src/test/resources/suites/data-provider-inheritance.xml new file mode 100644 index 000000000..4cc6ea8b0 --- /dev/null +++ b/allure-testng/src/test/resources/suites/data-provider-inheritance.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/allure-testng/src/test/resources/suites/data-provider-multiple-classes.xml b/allure-testng/src/test/resources/suites/data-provider-multiple-classes.xml new file mode 100644 index 000000000..a6686a985 --- /dev/null +++ b/allure-testng/src/test/resources/suites/data-provider-multiple-classes.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/allure-testng/src/test/resources/suites/data-provider-multiple-tests.xml b/allure-testng/src/test/resources/suites/data-provider-multiple-tests.xml new file mode 100644 index 000000000..e1b6dc1f8 --- /dev/null +++ b/allure-testng/src/test/resources/suites/data-provider-multiple-tests.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/allure-testng/src/test/resources/suites/data-provider-parallel.xml b/allure-testng/src/test/resources/suites/data-provider-parallel.xml new file mode 100644 index 000000000..5f0a94b2a --- /dev/null +++ b/allure-testng/src/test/resources/suites/data-provider-parallel.xml @@ -0,0 +1,8 @@ + + + + + + + + From e8068ef1be1d521fe0dcd18a5903fe5f2326806b Mon Sep 17 00:00:00 2001 From: Bartek Florczak Date: Tue, 27 Jan 2026 21:56:23 +0100 Subject: [PATCH 7/7] refactor: simplify assertions and improve readability in AllureTestNgTest Updated assertions to use more concise methods and removed unnecessary code. This enhances code clarity and maintainability. --- .../io/qameta/allure/testng/AllureTestNgTest.java | 14 ++++++-------- .../testng/samples/DataProviderParallel.java | 1 + 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index 8201ff595..c0cb226ca 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -39,7 +39,6 @@ import io.qameta.allure.testng.samples.TestsWithIdForFilter; import org.assertj.core.api.Condition; import org.assertj.core.groups.Tuple; -import org.testng.ITestNGListener; import org.testng.TestNG; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -130,7 +129,7 @@ public void shouldNotDisplayConfigurationFailsAsTests() { public void shouldSetConfigurationProperty() { AllureTestNgConfig allureTestNgConfig = AllureTestNgConfig.loadConfigProperties(); allureTestNgConfig.setHideDisabledTests(true); - assertThat(allureTestNgConfig.isHideDisabledTests()).isEqualTo(true); + assertThat(allureTestNgConfig.isHideDisabledTests()).isTrue(); } @AllureFeatures.Parallel @@ -1218,7 +1217,7 @@ public void shouldAddBeforeFixtureToFakeTestResult(final String suite, final Str .findAny(); assertThat(result).as("Before failed fake test result").isNotEmpty(); final Optional befores = results.getTestResultContainers().stream() - .filter(c -> Objects.nonNull(c.getBefores()) && c.getBefores().size() > 0) + .filter(c -> Objects.nonNull(c.getBefores()) && !c.getBefores().isEmpty()) .findAny(); assertThat(result).as("Before failed configuration container").isNotEmpty(); assertThat(befores.get().getChildren()) @@ -1252,7 +1251,6 @@ private AllureResults runTestNgSuites(AllureTestNgConfig config, final String... private AllureResults runTestNgSuites(final Consumer configurer, final String... suites) { - ; return runTestNgSuites(configurer, AllureTestNgConfig.loadConfigProperties(), suites); } @@ -1276,7 +1274,7 @@ private AllureResults runTestNgSuites(final Consumer configurer, new AllureTestNgTestFilter(), config); final TestNG testNg = new TestNG(false); - testNg.addListener((ITestNGListener) adapter); + testNg.addListener(adapter); testNg.setTestSuites(suiteFiles); configurer.accept(testNg); @@ -1461,7 +1459,7 @@ public void onlyId() { @Test @AllureFeatures.Filtration public void idAssignToOtherTest() { - TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(otherId)); + TestPlanV1_0 plan = new TestPlanV1_0().setTests(singletonList(otherId)); List testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults(); assertThat(testResults) @@ -1476,7 +1474,7 @@ public void idAssignToOtherTest() { @Test @AllureFeatures.Filtration public void skippedTest() { - TestPlanV1_0 plan = new TestPlanV1_0().setTests(Arrays.asList(skipped)); + TestPlanV1_0 plan = new TestPlanV1_0().setTests(singletonList(skipped)); List testResults = runTestPlan(plan, TestsWithIdForFilter.class).getTestResults(); assertThat(testResults) .hasSize(1) @@ -1508,7 +1506,7 @@ public AllureResults runTestPlan(final TestPlan plan, final Class... testClas return RunUtils.runTests(lifecycle -> { final AllureTestNg adapter = new AllureTestNg(lifecycle, new AllureTestNgTestFilter(plan)); final TestNG testNG = new TestNG(false); - testNG.addListener((ITestNGListener) adapter); + testNG.addListener(adapter); testNG.setTestClasses(testClasses); testNG.setOutputDirectory("build/test-output"); testNG.run(); diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java index 3f7c1bb67..4834229d5 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/DataProviderParallel.java @@ -27,5 +27,6 @@ public Object[][] dp() { @Test(dataProvider = "dp") public void test(String s) { + System.out.println("Test data: " + s); } }