From f675ae74b84da7ab3215fb84f92b1874e85d153a Mon Sep 17 00:00:00 2001 From: "Klare, Heiko" Date: Tue, 30 Dec 2025 00:49:37 +0100 Subject: [PATCH] Migrate performance session tests to JUnit 5 #903 The PerformanceSessionTestSuite and it's derived class and consumers still rely on JUnit 3 functionality. This change migrates the remaining performance session tests to the JUnit 5 SessionTestExecution. The SessionTestExecution is slightly extended to support the execution of repeated performance tests. The change also disentangles currently mixed up performance tests and makes each of them executable standalone. Contributes to https://github.com/eclipse-platform/eclipse.platform/issues/903 --- .../tests/harness/PerformanceTestRunner.java | 15 -- .../session/PerformanceSessionTest.java | 36 +++++ .../session/SessionTestExtensionHost.java | 29 ++++ .../samples/SamplePerformanceSessionTest.java | 61 +++++++++ .../core/tests/runtime/perf/AllPerfTests.java | 46 ++----- .../core/tests/runtime/perf/BenchPath.java | 37 +++-- .../perf/ContentTypePerformanceTest.java | 129 +++++++++--------- .../perf/PreferencePerformanceTest.java | 43 +++--- .../core/tests/runtime/perf/StartupTest.java | 38 +++++- .../tests/runtime/perf/UIStartupTest.java | 30 +++- 10 files changed, 299 insertions(+), 165 deletions(-) create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java create mode 100644 runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java index 396e004d81f..0cc915f42dd 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java @@ -38,21 +38,6 @@ public void setRegressionReason(String comment) { */ protected abstract void test() throws Exception; - /** - * Executes the performance test the given number of times. Use the outer time - * to execute the test several times in order to obtain a normalized average. Use - * the inner loop for very fast tests that would otherwise be difficult to measure - * due to Java's poor timer granularity. The inner loop is not needed for long - * tests that typically take more than a second to execute. - * - * @param testCase The test that is running (used to obtain an appropriate meter) - * @param outer The number of repetitions of the test. - * @param inner The number of repetitions within the performance timer. - */ - public final void run(TestCase testCase, int outer, int inner) { - run(testCase, null, outer, inner); - } - /** * Executes the performance test the given number of times. Use the outer time * to execute the test several times in order to obtain a normalized average. Use diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java new file mode 100644 index 00000000000..67a4432245c --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2025 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Test; + +/** + * A specialization of the {@link Test} annotation for marking a repeated + * performance session test. When executed with a {@link SessionTestExtension}, + * the specified number of repeated sessions will be executed. If of them gets + * the properties {@code eclipse.perf.dbloc} and {@code eclipse.perf.config} + * passed from the host. The last session also gets the + * {@code eclipse.perf.assertAgainst} property passed from the host to allow + * execution of performance assertions. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@Test +public @interface PerformanceSessionTest { + /** + * {@return the number of repeated sessions to execute} + */ + int repetitions() default 1; +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java index f654a59e27b..a961bf1a6c0 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java @@ -85,6 +85,35 @@ public void interceptTestMethod(Invocation invocation, ReflectiveInvocatio Method testMethod = extensionContext.getTestMethod().get(); boolean shouldFail = extensionContext.getTestMethod().get().getAnnotation(SessionShouldError.class) != null; + PerformanceSessionTest performanceSessionTestAnnotation = extensionContext.getTestMethod().get() + .getAnnotation(PerformanceSessionTest.class); + if (performanceSessionTestAnnotation != null) { + executePerformanceSessionTest(testClass, testMethod, shouldFail, + performanceSessionTestAnnotation.repetitions()); + } else { + exeuteSession(testClass, testMethod, shouldFail); + } + + } + + private void executePerformanceSessionTest(Class testClass, Method testMethod, boolean shouldFail, + int repetitions) throws Exception, Throwable { + try { + setSystemProperty("eclipse.perf.dbloc", System.getProperty("eclipse.perf.dbloc")); + setSystemProperty("eclipse.perf.config", System.getProperty("eclipse.perf.config")); + for (int i = 0; i < repetitions - 1; i++) { + exeuteSession(testClass, testMethod, shouldFail); + } + setSystemProperty("eclipse.perf.assertAgainst", System.getProperty("eclipse.perf.assertAgainst")); + exeuteSession(testClass, testMethod, shouldFail); + } finally { + setSystemProperty("eclipse.perf.assertAgainst", null); + setSystemProperty("eclipse.perf.dbloc", null); + setSystemProperty("eclipse.perf.config", null); + } + } + + private void exeuteSession(Class testClass, Method testMethod, boolean shouldFail) throws Exception, Throwable { try { prepareSession(); testExecutor.executeRemotely(testClass.getName(), testMethod.getName(), shouldFail); diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java new file mode 100644 index 00000000000..7d17e717114 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright (c) 2025 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session.samples; + +import static org.eclipse.core.tests.harness.TestHarnessPlugin.PI_HARNESS; + +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.test.performance.Dimension; +import org.eclipse.test.performance.Performance; +import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Examples demonstrating the behavior of performance session tests using the + * JUnit 5 platform. When executed, a warmup session is performed and five + * subsequent actual sessions with startup measurements will be performed. + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SamplePerformanceSessionTest { + @RegisterExtension + SessionTestExtension sessionTestExtension = SessionTestExtension.forPlugin(PI_HARNESS).create(); + + @Test + @Order(0) + public void warmup() { + actualTest(true); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(1) + public void runMeasurements() { + actualTest(false); + } + + private void actualTest(boolean warmup) { + PerformanceMeter meter = Performance.getDefault().createPerformanceMeter(getClass().getName() + ".startup"); + try { + meter.stop(); + if (!warmup) { + meter.commit(); + } + Performance.getDefault().assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -50, 5); + } finally { + meter.dispose(); + } + } + +} diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java index 543f85252c6..7f13cc42ec5 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java @@ -13,40 +13,16 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; -import junit.framework.*; -import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; -import org.eclipse.core.tests.session.*; -import org.eclipse.core.tests.session.SetupManager.SetupException; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; -public class AllPerfTests extends TestCase { - public static Test suite() { - TestSuite suite = new TestSuite(AllPerfTests.class.getName()); - - // make sure that the first run of the startup test is not recorded - it is heavily - // influenced by the presence and validity of the cached information - try { - PerformanceSessionTestSuite firstRun = new PerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 1, StartupTest.class); - Setup setup = firstRun.getSetup(); - setup.setSystemProperty("eclipseTest.ReportResults", "false"); - suite.addTest(firstRun); - } catch (SetupException e) { - fail("Unable to create warm up test"); - } - - // For this test to take advantage of the new runtime processing, we set "-eclipse.activateRuntimePlugins=false" - try { - PerformanceSessionTestSuite headlessSuite = new PerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 5, StartupTest.class); - Setup headlessSetup = headlessSuite.getSetup(); - headlessSetup.setSystemProperty("eclipse.activateRuntimePlugins", "false"); - suite.addTest(headlessSuite); - } catch (SetupException e) { - fail("Unable to setup headless startup performance test"); - } - - suite.addTest(new UIPerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 5, UIStartupTest.class)); - suite.addTestSuite(BenchPath.class); - suite.addTest(ContentTypePerformanceTest.suite()); - suite.addTestSuite(PreferencePerformanceTest.class); - return suite; - } +@Suite +@SelectClasses({ // + StartupTest.class, // + UIStartupTest.class, // + BenchPath.class, // + ContentTypePerformanceTest.class, // + PreferencePerformanceTest.class, // +}) +public class AllPerfTests { } diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java index 36cc872abc4..91346aaa6bf 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java @@ -14,25 +14,19 @@ package org.eclipse.core.tests.runtime.perf; import java.util.HashMap; -import junit.framework.TestCase; import org.eclipse.core.runtime.IPath; import org.eclipse.core.tests.harness.PerformanceTestRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -public class BenchPath extends TestCase { - - public BenchPath() { - super(); - } - - public BenchPath(String testName) { - super(testName); - } +public class BenchPath { /** - * Tests performance of equals and hashCode by using paths - * as keys in a hash map. + * Tests performance of equals and hashCode by using paths as keys in a hash + * map. */ - public void testHash() { + @Test + public void testHash(TestInfo testInfo) throws Exception { final int REPEAT = 500000; final IPath[] paths = generateVariousPaths(); final HashMap map = new HashMap<>(32); @@ -47,13 +41,14 @@ protected void test() { map.get(paths[p]); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** * Tests the performance of path creation */ - public void testPathCreation() { + @Test + public void testPathCreation(TestInfo testInfo) throws Exception { final int REPEAT = 50000; new PerformanceTestRunner() { @Override @@ -82,13 +77,14 @@ protected void test() { IPath.fromOSString("/Foo/bar/baz/blap/blam/abc.txt"); IPath.fromOSString("/Foo/bar/baz/blap/blam/blip/boop/abc.txt"); } - }.run(this, 20, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 20, REPEAT); } /** * Tests the performance of Path.toOSString */ - public void testToOSString() { + @Test + public void testToOSString(TestInfo testInfo) throws Exception { final int REPEAT = 50000; final IPath[] paths = generateVariousPaths(); new PerformanceTestRunner() { @@ -98,13 +94,14 @@ protected void test() { paths[p].toOSString(); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** * Tests the performance of Path.toOSString */ - public void testToString() { + @Test + public void testToString(TestInfo testInfo) throws Exception { final int REPEAT = 50000; final IPath[] paths = generateVariousPaths(); new PerformanceTestRunner() { @@ -114,7 +111,7 @@ protected void test() { paths[p].toString(); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java index 8d6c829f67b..aa857a3bd9c 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java @@ -18,6 +18,9 @@ import static org.eclipse.core.tests.harness.FileSystemHelper.clear; import static org.eclipse.core.tests.harness.FileSystemHelper.getTempDir; import static org.eclipse.core.tests.runtime.RuntimeTestsPlugin.PI_RUNTIME_TESTS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -25,9 +28,6 @@ import java.io.IOException; import java.io.Writer; import java.net.URL; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; import org.eclipse.core.internal.content.ContentTypeBuilder; import org.eclipse.core.internal.content.ContentTypeHandler; import org.eclipse.core.internal.content.Util; @@ -40,14 +40,26 @@ import org.eclipse.core.tests.harness.BundleTestingHelper; import org.eclipse.core.tests.harness.PerformanceTestRunner; import org.eclipse.core.tests.harness.TestRegistryChangeListener; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; -import org.eclipse.core.tests.session.PerformanceSessionTestSuite; -import org.eclipse.core.tests.session.SessionTestSuite; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; @SuppressWarnings("restriction") -public class ContentTypePerformanceTest extends TestCase { +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ContentTypePerformanceTest { + + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS).create(); private final static String CONTENT_TYPE_PREF_NODE = Platform.PI_RUNTIME + IPath.SEPARATOR + "content-types"; //$NON-NLS-1$ private static final String DEFAULT_NAME = "file_" + ContentTypePerformanceTest.class.getName(); @@ -139,39 +151,6 @@ private static String getSignatureString(int number) { return result.toString(); } - public static Test suite() { - TestSuite suite = new TestSuite(ContentTypePerformanceTest.class.getName()); - - // suite.addTest(new ContentTypePerformanceTest("testDoSetUp")); - // suite.addTest(new ContentTypePerformanceTest("testContentMatching")); - // suite.addTest(new ContentTypePerformanceTest("testContentTXTMatching")); - // suite.addTest(new ContentTypePerformanceTest("testContentXMLMatching")); - // suite.addTest(new ContentTypePerformanceTest("testDoTearDown")); - - SessionTestSuite setUp = new SessionTestSuite(PI_RUNTIME_TESTS, "testDoSetUp"); - setUp.addTest(new ContentTypePerformanceTest("testDoSetUp")); - suite.addTest(setUp); - - TestSuite singleRun = new PerformanceSessionTestSuite(PI_RUNTIME_TESTS, 1, "singleSessionTests"); - singleRun.addTest(new ContentTypePerformanceTest("testContentMatching")); - singleRun.addTest(new ContentTypePerformanceTest("testNameMatching")); - singleRun.addTest(new ContentTypePerformanceTest("testIsKindOf")); - suite.addTest(singleRun); - - TestSuite loadCatalog = new PerformanceSessionTestSuite(PI_RUNTIME_TESTS, 10, "multipleSessionTests"); - loadCatalog.addTest(new ContentTypePerformanceTest("testLoadCatalog")); - suite.addTest(loadCatalog); - - TestSuite tearDown = new SessionTestSuite(PI_RUNTIME_TESTS, "testDoTearDown"); - tearDown.addTest(new ContentTypePerformanceTest("testDoTearDown")); - suite.addTest(tearDown); - return suite; - } - - public ContentTypePerformanceTest(String name) { - super(name); - } - private int countTestContentTypes(IContentType[] all) { String namespace = TEST_DATA_ID + '.'; int count = 0; @@ -183,11 +162,11 @@ private int countTestContentTypes(IContentType[] all) { return count; } - public IPath getExtraPluginLocation() { + public static IPath getExtraPluginLocation() { return getTempDir().append(TEST_DATA_ID); } - private Bundle installContentTypes(String tag, int numberOfLevels, int nodesPerLevel) + private static Bundle installContentTypes(int numberOfLevels, int nodesPerLevel) throws IOException, BundleException { TestRegistryChangeListener listener = new TestRegistryChangeListener(Platform.PI_RUNTIME, ContentTypeBuilder.PT_CONTENTTYPES, null, null); Bundle installed = null; @@ -225,7 +204,7 @@ private Bundle installContentTypes(String tag, int numberOfLevels, int nodesPerL } installed = RuntimeTestsPlugin.getContext().installBundle(installURL.toExternalForm()); BundleTestingHelper.refreshPackages(RuntimeTestsPlugin.getContext(), new Bundle[] {installed}); - assertTrue(tag + ".4.0", listener.eventReceived(10000)); + assertTrue(listener.eventReceived(10000)); } finally { listener.unregister(); } @@ -273,8 +252,13 @@ private void loadPreferences() { InstanceScope.INSTANCE.getNode(CONTENT_TYPE_PREF_NODE); } - /** Tests how much the size of the catalog affects the performance of content type matching by content analysis */ - public void testContentMatching() { + /** + * Tests how much the size of the catalog affects the performance of content + * type matching by content analysis + */ + @PerformanceSessionTest + @Order(0) + public void testContentMatching() throws Exception { loadPreferences(); // warm up content type registry final IContentTypeManager manager = loadContentTypeManager(); @@ -295,52 +279,54 @@ protected void test() { .satisfies(it -> assertThat(it.getId()).as("id").isEqualTo(id)); } } - }.run(this, 10, 2); + }.run(getClass(), "testContentMatching", 10, 2); } - @Override - protected void setUp() throws Exception { - super.setUp(); - if (getName().equals("testDoSetUp") || getName().equals("testDoTearDown")) { - return; - } + @BeforeEach + void setUp() throws Exception { Bundle installed = RuntimeTestsPlugin.getContext() .installBundle(getExtraPluginLocation().toFile().toURI().toURL().toExternalForm()); BundleTestingHelper.refreshPackages(RuntimeTestsPlugin.getContext(), new Bundle[] { installed }); } - public void testDoSetUp() throws IOException, BundleException { - installContentTypes("1.0", NUMBER_OF_LEVELS, ELEMENTS_PER_LEVEL); + @BeforeAll + static void doSetUp() throws IOException, BundleException { + installContentTypes(NUMBER_OF_LEVELS, ELEMENTS_PER_LEVEL); } - public void testDoTearDown() { + @AfterAll + static void doTearDown() { clear(getExtraPluginLocation().toFile()); } - public void testIsKindOf() { + @PerformanceSessionTest + @Order(2) + public void testIsKindOf() throws Exception { // warm up preference service loadPreferences(); // warm up content type registry final IContentTypeManager manager = loadContentTypeManager(); loadChildren(); final IContentType root = manager.getContentType(getContentTypeId(0)); - assertNotNull("2.0", root); + assertNotNull(root); new PerformanceTestRunner() { @Override protected void test() { for (int i = 0; i < TOTAL_NUMBER_OF_ELEMENTS; i++) { IContentType type = manager.getContentType(getContentTypeId(i)); - assertNotNull("3.0." + i, type); - assertTrue("3.1." + i, type.isKindOf(root)); + assertNotNull(type, "element is null: " + i); + assertTrue(type.isKindOf(root), "element is of wrong type: " + i); } } - }.run(this, 10, 500); + }.run(getClass(), "testIsKindOf", 10, 500); } /** * This test is intended for running as a session test. */ - public void testLoadCatalog() { + @PerformanceSessionTest(repetitions = 10) + @Order(3) + public void testLoadCatalog() throws Exception { // warm up preference service loadPreferences(); PerformanceTestRunner runner = new PerformanceTestRunner() { @@ -350,13 +336,22 @@ protected void test() { Platform.getContentTypeManager().getContentType(IContentTypeManager.CT_TEXT); } }; - runner.run(this, 1, /* must run only once - the suite controls how many sessions are run */1); + runner.run(getClass(), "testLoadCatalog", 1, + /* must run only once - the suite controls how many sessions are run */1); // sanity check to make sure we are running with good data - assertEquals("missing content types", TOTAL_NUMBER_OF_ELEMENTS, countTestContentTypes(Platform.getContentTypeManager().getAllContentTypes())); + assertEquals(TOTAL_NUMBER_OF_ELEMENTS, + countTestContentTypes(Platform.getContentTypeManager().getAllContentTypes()), "missing content types"); } - /** Tests how much the size of the catalog affects the performance of content type matching by name */ - public void testNameMatching() { + /** + * Tests how much the size of the catalog affects the performance of content + * type matching by name + * + * @throws Exception + */ + @PerformanceSessionTest + @Order(1) + public void testNameMatching() throws Exception { // warm up preference service loadPreferences(); // warm up content type registry @@ -368,10 +363,10 @@ public void testNameMatching() { protected void test() { IContentType[] associated = manager.findContentTypesFor("foo.txt"); // we know at least the etxt content type should be here - assertTrue("2.0", associated.length >= 1); + assertTrue(associated.length >= 1); // and it is supposed to be the first one (since it is at the root) - assertEquals("2.1", IContentTypeManager.CT_TEXT, associated[0].getId()); + assertEquals(IContentTypeManager.CT_TEXT, associated[0].getId()); } - }.run(this, 10, 200000); + }.run(getClass(), "testNameMatching", 10, 200000); } } \ No newline at end of file diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java index 5d0c78144a9..c8739f5a7b6 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java @@ -15,26 +15,19 @@ import java.util.ArrayList; import java.util.UUID; -import junit.framework.TestCase; import org.eclipse.core.runtime.Platform; import org.eclipse.core.runtime.preferences.IEclipsePreferences; import org.eclipse.core.tests.harness.PerformanceTestRunner; import org.eclipse.core.tests.internal.preferences.TestScope; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.osgi.service.prefs.BackingStoreException; import org.osgi.service.prefs.Preferences; -public class PreferencePerformanceTest extends TestCase { +public class PreferencePerformanceTest { private static final int INNER_LOOP = 10000; private static final int KEYS_PER_NODE = 1000; - public PreferencePerformanceTest() { - super(); - } - - public PreferencePerformanceTest(String testName) { - super(testName); - } - /* * Return a 2 dimensional String array with the first element being the keys * and the second being the values. All the keys will have the given prefix. @@ -95,7 +88,8 @@ private String[][] getUniqueKeys(int size) { * This is a good finger print test because preference keys typically have a common * prefix (org.eclipse.component.keyName). */ - public void testGetStringCommonPrefixKeys() { + @Test + public void testGetStringCommonPrefixKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getCommonPrefixKeys(KEYS_PER_NODE, qualifier); @@ -130,14 +124,15 @@ protected void test() { } }; runner.setFingerprintName("Retrieve preference values"); - runner.run(this, 10, INNER_LOOP); + runner.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to get KEYS_PER_NODE keys that aren't there. * Fill the node up with KEYS_PER_NODE key/value pairs so it has some data */ - public void testGetStringMisses() { + @Test + public void testGetStringMisses(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getUniqueKeys(KEYS_PER_NODE); @@ -171,14 +166,15 @@ protected void test() { prefs.get(missingKeys[i], null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to retrieve KEYS_PER_NODE keys which are constructed * from sequential integers. */ - public void testGetStringSequentialKeys() { + @Test + public void testGetStringSequentialKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getSequentialKeys(KEYS_PER_NODE); @@ -211,13 +207,14 @@ protected void test() { prefs.get(key, null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to get KEYS_PER_NODE keys that are unique. */ - public void testGetStringUniqueKeys() { + @Test + public void testGetStringUniqueKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getUniqueKeys(KEYS_PER_NODE); @@ -250,13 +247,14 @@ protected void test() { prefs.get(key, null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to put KEYS_PER_NODE keys into a preference node. */ - public void testPutStringKeys() { + @Test + public void testPutStringKeys(TestInfo testInfo) throws Exception { // setup outside the timed block final String qualifier = getUniqueString(); @@ -291,13 +289,14 @@ protected void test() { prefs.put(keys[i], values[i]); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Add KEYS_PER_NODE keys to a preference node and then remove them one at a time. */ - public void testRemoveStringKeys() { + @Test + public void testRemoveStringKeys(TestInfo testInfo) throws Exception { // gather the key/value pairs before so we don't time it final String qualifier = getUniqueString(); @@ -338,7 +337,7 @@ protected void test() { prefs.remove(key); } } - }.run(this, 50, 1); + }.run(getClass(), testInfo.getDisplayName(), 50, 1); } private String getUniqueString() { diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java index c9531478509..c5ac362cc2b 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java @@ -13,15 +13,46 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; +import org.eclipse.core.tests.harness.session.ExecuteInHost; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.Performance; import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class StartupTest { + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS).create(); + @Test - public void testApplicationStartup() { + @Order(0) + public void warmup() { + testApplicationStartup(true); + } + + @Test + @ExecuteInHost + @Order(1) + public void activateRuntimePlugins() { + sessionTestExtension.setSystemProperty("eclipse.activateRuntimePlugins", "false"); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(2) + public void runMeasurements() { + testApplicationStartup(false); + } + + private void testApplicationStartup(boolean warmup) { PerformanceMeter meter = Performance.getDefault() .createPerformanceMeter(getClass().getName() + '.' + StartupTest.class.getName()); try { @@ -29,9 +60,7 @@ public void testApplicationStartup() { // tag for showing in the performance fingerprint graph Performance performance = Performance.getDefault(); performance.tagAsGlobalSummary(meter, "Core Headless Startup", Dimension.ELAPSED_PROCESS); - String reportOption = System.getProperty("eclipseTest.ReportResults"); - boolean bReport = (reportOption == null) ? true : !("false".equalsIgnoreCase(reportOption)); - if (bReport) { + if (!warmup) { meter.commit(); } Performance.getDefault().assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -100, 5); @@ -39,4 +68,5 @@ public void testApplicationStartup() { meter.dispose(); } } + } diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java index 50a6104ac88..755ef9716bd 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java @@ -13,22 +13,48 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.Performance; import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class UIStartupTest { + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS) + .withApplicationId(SessionTestExtension.UI_TEST_APPLICATION).create(); + @Test - public void testUIApplicationStartup() { + @Order(0) + public void warmup() { + testUIApplicationStartup(true); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(1) + public void runMeasurements() { + testUIApplicationStartup(false); + } + + public void testUIApplicationStartup(boolean warmup) { PerformanceMeter meter = Performance.getDefault() .createPerformanceMeter(getClass().getName() + '.' + UIStartupTest.class.getName()); try { meter.stop(); Performance performance = Performance.getDefault(); performance.tagAsGlobalSummary(meter, "Core UI Startup", Dimension.ELAPSED_PROCESS); - meter.commit(); + if (!warmup) { + meter.commit(); + } performance.assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -50, 5); } finally { meter.dispose();