diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java index 396e004d81f..0cc915f42dd 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/PerformanceTestRunner.java @@ -38,21 +38,6 @@ public void setRegressionReason(String comment) { */ protected abstract void test() throws Exception; - /** - * Executes the performance test the given number of times. Use the outer time - * to execute the test several times in order to obtain a normalized average. Use - * the inner loop for very fast tests that would otherwise be difficult to measure - * due to Java's poor timer granularity. The inner loop is not needed for long - * tests that typically take more than a second to execute. - * - * @param testCase The test that is running (used to obtain an appropriate meter) - * @param outer The number of repetitions of the test. - * @param inner The number of repetitions within the performance timer. - */ - public final void run(TestCase testCase, int outer, int inner) { - run(testCase, null, outer, inner); - } - /** * Executes the performance test the given number of times. Use the outer time * to execute the test several times in order to obtain a normalized average. Use diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java new file mode 100644 index 00000000000..67a4432245c --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/PerformanceSessionTest.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2025 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Test; + +/** + * A specialization of the {@link Test} annotation for marking a repeated + * performance session test. When executed with a {@link SessionTestExtension}, + * the specified number of repeated sessions will be executed. If of them gets + * the properties {@code eclipse.perf.dbloc} and {@code eclipse.perf.config} + * passed from the host. The last session also gets the + * {@code eclipse.perf.assertAgainst} property passed from the host to allow + * execution of performance assertions. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@Test +public @interface PerformanceSessionTest { + /** + * {@return the number of repeated sessions to execute} + */ + int repetitions() default 1; +} diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java index f654a59e27b..a961bf1a6c0 100644 --- a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/SessionTestExtensionHost.java @@ -85,6 +85,35 @@ public void interceptTestMethod(Invocation invocation, ReflectiveInvocatio Method testMethod = extensionContext.getTestMethod().get(); boolean shouldFail = extensionContext.getTestMethod().get().getAnnotation(SessionShouldError.class) != null; + PerformanceSessionTest performanceSessionTestAnnotation = extensionContext.getTestMethod().get() + .getAnnotation(PerformanceSessionTest.class); + if (performanceSessionTestAnnotation != null) { + executePerformanceSessionTest(testClass, testMethod, shouldFail, + performanceSessionTestAnnotation.repetitions()); + } else { + exeuteSession(testClass, testMethod, shouldFail); + } + + } + + private void executePerformanceSessionTest(Class testClass, Method testMethod, boolean shouldFail, + int repetitions) throws Exception, Throwable { + try { + setSystemProperty("eclipse.perf.dbloc", System.getProperty("eclipse.perf.dbloc")); + setSystemProperty("eclipse.perf.config", System.getProperty("eclipse.perf.config")); + for (int i = 0; i < repetitions - 1; i++) { + exeuteSession(testClass, testMethod, shouldFail); + } + setSystemProperty("eclipse.perf.assertAgainst", System.getProperty("eclipse.perf.assertAgainst")); + exeuteSession(testClass, testMethod, shouldFail); + } finally { + setSystemProperty("eclipse.perf.assertAgainst", null); + setSystemProperty("eclipse.perf.dbloc", null); + setSystemProperty("eclipse.perf.config", null); + } + } + + private void exeuteSession(Class testClass, Method testMethod, boolean shouldFail) throws Exception, Throwable { try { prepareSession(); testExecutor.executeRemotely(testClass.getName(), testMethod.getName(), shouldFail); diff --git a/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java new file mode 100644 index 00000000000..7d17e717114 --- /dev/null +++ b/runtime/tests/org.eclipse.core.tests.harness/src/org/eclipse/core/tests/harness/session/samples/SamplePerformanceSessionTest.java @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright (c) 2025 Vector Informatik GmbH and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.core.tests.harness.session.samples; + +import static org.eclipse.core.tests.harness.TestHarnessPlugin.PI_HARNESS; + +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.test.performance.Dimension; +import org.eclipse.test.performance.Performance; +import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Examples demonstrating the behavior of performance session tests using the + * JUnit 5 platform. When executed, a warmup session is performed and five + * subsequent actual sessions with startup measurements will be performed. + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SamplePerformanceSessionTest { + @RegisterExtension + SessionTestExtension sessionTestExtension = SessionTestExtension.forPlugin(PI_HARNESS).create(); + + @Test + @Order(0) + public void warmup() { + actualTest(true); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(1) + public void runMeasurements() { + actualTest(false); + } + + private void actualTest(boolean warmup) { + PerformanceMeter meter = Performance.getDefault().createPerformanceMeter(getClass().getName() + ".startup"); + try { + meter.stop(); + if (!warmup) { + meter.commit(); + } + Performance.getDefault().assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -50, 5); + } finally { + meter.dispose(); + } + } + +} diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java index 543f85252c6..7f13cc42ec5 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/AllPerfTests.java @@ -13,40 +13,16 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; -import junit.framework.*; -import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; -import org.eclipse.core.tests.session.*; -import org.eclipse.core.tests.session.SetupManager.SetupException; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; -public class AllPerfTests extends TestCase { - public static Test suite() { - TestSuite suite = new TestSuite(AllPerfTests.class.getName()); - - // make sure that the first run of the startup test is not recorded - it is heavily - // influenced by the presence and validity of the cached information - try { - PerformanceSessionTestSuite firstRun = new PerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 1, StartupTest.class); - Setup setup = firstRun.getSetup(); - setup.setSystemProperty("eclipseTest.ReportResults", "false"); - suite.addTest(firstRun); - } catch (SetupException e) { - fail("Unable to create warm up test"); - } - - // For this test to take advantage of the new runtime processing, we set "-eclipse.activateRuntimePlugins=false" - try { - PerformanceSessionTestSuite headlessSuite = new PerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 5, StartupTest.class); - Setup headlessSetup = headlessSuite.getSetup(); - headlessSetup.setSystemProperty("eclipse.activateRuntimePlugins", "false"); - suite.addTest(headlessSuite); - } catch (SetupException e) { - fail("Unable to setup headless startup performance test"); - } - - suite.addTest(new UIPerformanceSessionTestSuite(RuntimeTestsPlugin.PI_RUNTIME_TESTS, 5, UIStartupTest.class)); - suite.addTestSuite(BenchPath.class); - suite.addTest(ContentTypePerformanceTest.suite()); - suite.addTestSuite(PreferencePerformanceTest.class); - return suite; - } +@Suite +@SelectClasses({ // + StartupTest.class, // + UIStartupTest.class, // + BenchPath.class, // + ContentTypePerformanceTest.class, // + PreferencePerformanceTest.class, // +}) +public class AllPerfTests { } diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java index 36cc872abc4..91346aaa6bf 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/BenchPath.java @@ -14,25 +14,19 @@ package org.eclipse.core.tests.runtime.perf; import java.util.HashMap; -import junit.framework.TestCase; import org.eclipse.core.runtime.IPath; import org.eclipse.core.tests.harness.PerformanceTestRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; -public class BenchPath extends TestCase { - - public BenchPath() { - super(); - } - - public BenchPath(String testName) { - super(testName); - } +public class BenchPath { /** - * Tests performance of equals and hashCode by using paths - * as keys in a hash map. + * Tests performance of equals and hashCode by using paths as keys in a hash + * map. */ - public void testHash() { + @Test + public void testHash(TestInfo testInfo) throws Exception { final int REPEAT = 500000; final IPath[] paths = generateVariousPaths(); final HashMap map = new HashMap<>(32); @@ -47,13 +41,14 @@ protected void test() { map.get(paths[p]); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** * Tests the performance of path creation */ - public void testPathCreation() { + @Test + public void testPathCreation(TestInfo testInfo) throws Exception { final int REPEAT = 50000; new PerformanceTestRunner() { @Override @@ -82,13 +77,14 @@ protected void test() { IPath.fromOSString("/Foo/bar/baz/blap/blam/abc.txt"); IPath.fromOSString("/Foo/bar/baz/blap/blam/blip/boop/abc.txt"); } - }.run(this, 20, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 20, REPEAT); } /** * Tests the performance of Path.toOSString */ - public void testToOSString() { + @Test + public void testToOSString(TestInfo testInfo) throws Exception { final int REPEAT = 50000; final IPath[] paths = generateVariousPaths(); new PerformanceTestRunner() { @@ -98,13 +94,14 @@ protected void test() { paths[p].toOSString(); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** * Tests the performance of Path.toOSString */ - public void testToString() { + @Test + public void testToString(TestInfo testInfo) throws Exception { final int REPEAT = 50000; final IPath[] paths = generateVariousPaths(); new PerformanceTestRunner() { @@ -114,7 +111,7 @@ protected void test() { paths[p].toString(); } } - }.run(this, 10, REPEAT); + }.run(getClass(), testInfo.getDisplayName(), 10, REPEAT); } /** diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java index 8d6c829f67b..aa857a3bd9c 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/ContentTypePerformanceTest.java @@ -18,6 +18,9 @@ import static org.eclipse.core.tests.harness.FileSystemHelper.clear; import static org.eclipse.core.tests.harness.FileSystemHelper.getTempDir; import static org.eclipse.core.tests.runtime.RuntimeTestsPlugin.PI_RUNTIME_TESTS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -25,9 +28,6 @@ import java.io.IOException; import java.io.Writer; import java.net.URL; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; import org.eclipse.core.internal.content.ContentTypeBuilder; import org.eclipse.core.internal.content.ContentTypeHandler; import org.eclipse.core.internal.content.Util; @@ -40,14 +40,26 @@ import org.eclipse.core.tests.harness.BundleTestingHelper; import org.eclipse.core.tests.harness.PerformanceTestRunner; import org.eclipse.core.tests.harness.TestRegistryChangeListener; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; -import org.eclipse.core.tests.session.PerformanceSessionTestSuite; -import org.eclipse.core.tests.session.SessionTestSuite; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; @SuppressWarnings("restriction") -public class ContentTypePerformanceTest extends TestCase { +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ContentTypePerformanceTest { + + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS).create(); private final static String CONTENT_TYPE_PREF_NODE = Platform.PI_RUNTIME + IPath.SEPARATOR + "content-types"; //$NON-NLS-1$ private static final String DEFAULT_NAME = "file_" + ContentTypePerformanceTest.class.getName(); @@ -139,39 +151,6 @@ private static String getSignatureString(int number) { return result.toString(); } - public static Test suite() { - TestSuite suite = new TestSuite(ContentTypePerformanceTest.class.getName()); - - // suite.addTest(new ContentTypePerformanceTest("testDoSetUp")); - // suite.addTest(new ContentTypePerformanceTest("testContentMatching")); - // suite.addTest(new ContentTypePerformanceTest("testContentTXTMatching")); - // suite.addTest(new ContentTypePerformanceTest("testContentXMLMatching")); - // suite.addTest(new ContentTypePerformanceTest("testDoTearDown")); - - SessionTestSuite setUp = new SessionTestSuite(PI_RUNTIME_TESTS, "testDoSetUp"); - setUp.addTest(new ContentTypePerformanceTest("testDoSetUp")); - suite.addTest(setUp); - - TestSuite singleRun = new PerformanceSessionTestSuite(PI_RUNTIME_TESTS, 1, "singleSessionTests"); - singleRun.addTest(new ContentTypePerformanceTest("testContentMatching")); - singleRun.addTest(new ContentTypePerformanceTest("testNameMatching")); - singleRun.addTest(new ContentTypePerformanceTest("testIsKindOf")); - suite.addTest(singleRun); - - TestSuite loadCatalog = new PerformanceSessionTestSuite(PI_RUNTIME_TESTS, 10, "multipleSessionTests"); - loadCatalog.addTest(new ContentTypePerformanceTest("testLoadCatalog")); - suite.addTest(loadCatalog); - - TestSuite tearDown = new SessionTestSuite(PI_RUNTIME_TESTS, "testDoTearDown"); - tearDown.addTest(new ContentTypePerformanceTest("testDoTearDown")); - suite.addTest(tearDown); - return suite; - } - - public ContentTypePerformanceTest(String name) { - super(name); - } - private int countTestContentTypes(IContentType[] all) { String namespace = TEST_DATA_ID + '.'; int count = 0; @@ -183,11 +162,11 @@ private int countTestContentTypes(IContentType[] all) { return count; } - public IPath getExtraPluginLocation() { + public static IPath getExtraPluginLocation() { return getTempDir().append(TEST_DATA_ID); } - private Bundle installContentTypes(String tag, int numberOfLevels, int nodesPerLevel) + private static Bundle installContentTypes(int numberOfLevels, int nodesPerLevel) throws IOException, BundleException { TestRegistryChangeListener listener = new TestRegistryChangeListener(Platform.PI_RUNTIME, ContentTypeBuilder.PT_CONTENTTYPES, null, null); Bundle installed = null; @@ -225,7 +204,7 @@ private Bundle installContentTypes(String tag, int numberOfLevels, int nodesPerL } installed = RuntimeTestsPlugin.getContext().installBundle(installURL.toExternalForm()); BundleTestingHelper.refreshPackages(RuntimeTestsPlugin.getContext(), new Bundle[] {installed}); - assertTrue(tag + ".4.0", listener.eventReceived(10000)); + assertTrue(listener.eventReceived(10000)); } finally { listener.unregister(); } @@ -273,8 +252,13 @@ private void loadPreferences() { InstanceScope.INSTANCE.getNode(CONTENT_TYPE_PREF_NODE); } - /** Tests how much the size of the catalog affects the performance of content type matching by content analysis */ - public void testContentMatching() { + /** + * Tests how much the size of the catalog affects the performance of content + * type matching by content analysis + */ + @PerformanceSessionTest + @Order(0) + public void testContentMatching() throws Exception { loadPreferences(); // warm up content type registry final IContentTypeManager manager = loadContentTypeManager(); @@ -295,52 +279,54 @@ protected void test() { .satisfies(it -> assertThat(it.getId()).as("id").isEqualTo(id)); } } - }.run(this, 10, 2); + }.run(getClass(), "testContentMatching", 10, 2); } - @Override - protected void setUp() throws Exception { - super.setUp(); - if (getName().equals("testDoSetUp") || getName().equals("testDoTearDown")) { - return; - } + @BeforeEach + void setUp() throws Exception { Bundle installed = RuntimeTestsPlugin.getContext() .installBundle(getExtraPluginLocation().toFile().toURI().toURL().toExternalForm()); BundleTestingHelper.refreshPackages(RuntimeTestsPlugin.getContext(), new Bundle[] { installed }); } - public void testDoSetUp() throws IOException, BundleException { - installContentTypes("1.0", NUMBER_OF_LEVELS, ELEMENTS_PER_LEVEL); + @BeforeAll + static void doSetUp() throws IOException, BundleException { + installContentTypes(NUMBER_OF_LEVELS, ELEMENTS_PER_LEVEL); } - public void testDoTearDown() { + @AfterAll + static void doTearDown() { clear(getExtraPluginLocation().toFile()); } - public void testIsKindOf() { + @PerformanceSessionTest + @Order(2) + public void testIsKindOf() throws Exception { // warm up preference service loadPreferences(); // warm up content type registry final IContentTypeManager manager = loadContentTypeManager(); loadChildren(); final IContentType root = manager.getContentType(getContentTypeId(0)); - assertNotNull("2.0", root); + assertNotNull(root); new PerformanceTestRunner() { @Override protected void test() { for (int i = 0; i < TOTAL_NUMBER_OF_ELEMENTS; i++) { IContentType type = manager.getContentType(getContentTypeId(i)); - assertNotNull("3.0." + i, type); - assertTrue("3.1." + i, type.isKindOf(root)); + assertNotNull(type, "element is null: " + i); + assertTrue(type.isKindOf(root), "element is of wrong type: " + i); } } - }.run(this, 10, 500); + }.run(getClass(), "testIsKindOf", 10, 500); } /** * This test is intended for running as a session test. */ - public void testLoadCatalog() { + @PerformanceSessionTest(repetitions = 10) + @Order(3) + public void testLoadCatalog() throws Exception { // warm up preference service loadPreferences(); PerformanceTestRunner runner = new PerformanceTestRunner() { @@ -350,13 +336,22 @@ protected void test() { Platform.getContentTypeManager().getContentType(IContentTypeManager.CT_TEXT); } }; - runner.run(this, 1, /* must run only once - the suite controls how many sessions are run */1); + runner.run(getClass(), "testLoadCatalog", 1, + /* must run only once - the suite controls how many sessions are run */1); // sanity check to make sure we are running with good data - assertEquals("missing content types", TOTAL_NUMBER_OF_ELEMENTS, countTestContentTypes(Platform.getContentTypeManager().getAllContentTypes())); + assertEquals(TOTAL_NUMBER_OF_ELEMENTS, + countTestContentTypes(Platform.getContentTypeManager().getAllContentTypes()), "missing content types"); } - /** Tests how much the size of the catalog affects the performance of content type matching by name */ - public void testNameMatching() { + /** + * Tests how much the size of the catalog affects the performance of content + * type matching by name + * + * @throws Exception + */ + @PerformanceSessionTest + @Order(1) + public void testNameMatching() throws Exception { // warm up preference service loadPreferences(); // warm up content type registry @@ -368,10 +363,10 @@ public void testNameMatching() { protected void test() { IContentType[] associated = manager.findContentTypesFor("foo.txt"); // we know at least the etxt content type should be here - assertTrue("2.0", associated.length >= 1); + assertTrue(associated.length >= 1); // and it is supposed to be the first one (since it is at the root) - assertEquals("2.1", IContentTypeManager.CT_TEXT, associated[0].getId()); + assertEquals(IContentTypeManager.CT_TEXT, associated[0].getId()); } - }.run(this, 10, 200000); + }.run(getClass(), "testNameMatching", 10, 200000); } } \ No newline at end of file diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java index 5d0c78144a9..c8739f5a7b6 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/PreferencePerformanceTest.java @@ -15,26 +15,19 @@ import java.util.ArrayList; import java.util.UUID; -import junit.framework.TestCase; import org.eclipse.core.runtime.Platform; import org.eclipse.core.runtime.preferences.IEclipsePreferences; import org.eclipse.core.tests.harness.PerformanceTestRunner; import org.eclipse.core.tests.internal.preferences.TestScope; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.osgi.service.prefs.BackingStoreException; import org.osgi.service.prefs.Preferences; -public class PreferencePerformanceTest extends TestCase { +public class PreferencePerformanceTest { private static final int INNER_LOOP = 10000; private static final int KEYS_PER_NODE = 1000; - public PreferencePerformanceTest() { - super(); - } - - public PreferencePerformanceTest(String testName) { - super(testName); - } - /* * Return a 2 dimensional String array with the first element being the keys * and the second being the values. All the keys will have the given prefix. @@ -95,7 +88,8 @@ private String[][] getUniqueKeys(int size) { * This is a good finger print test because preference keys typically have a common * prefix (org.eclipse.component.keyName). */ - public void testGetStringCommonPrefixKeys() { + @Test + public void testGetStringCommonPrefixKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getCommonPrefixKeys(KEYS_PER_NODE, qualifier); @@ -130,14 +124,15 @@ protected void test() { } }; runner.setFingerprintName("Retrieve preference values"); - runner.run(this, 10, INNER_LOOP); + runner.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to get KEYS_PER_NODE keys that aren't there. * Fill the node up with KEYS_PER_NODE key/value pairs so it has some data */ - public void testGetStringMisses() { + @Test + public void testGetStringMisses(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getUniqueKeys(KEYS_PER_NODE); @@ -171,14 +166,15 @@ protected void test() { prefs.get(missingKeys[i], null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to retrieve KEYS_PER_NODE keys which are constructed * from sequential integers. */ - public void testGetStringSequentialKeys() { + @Test + public void testGetStringSequentialKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getSequentialKeys(KEYS_PER_NODE); @@ -211,13 +207,14 @@ protected void test() { prefs.get(key, null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to get KEYS_PER_NODE keys that are unique. */ - public void testGetStringUniqueKeys() { + @Test + public void testGetStringUniqueKeys(TestInfo testInfo) throws Exception { // setup final String qualifier = getUniqueString(); String[][] kvp = getUniqueKeys(KEYS_PER_NODE); @@ -250,13 +247,14 @@ protected void test() { prefs.get(key, null); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Time how long it takes to put KEYS_PER_NODE keys into a preference node. */ - public void testPutStringKeys() { + @Test + public void testPutStringKeys(TestInfo testInfo) throws Exception { // setup outside the timed block final String qualifier = getUniqueString(); @@ -291,13 +289,14 @@ protected void test() { prefs.put(keys[i], values[i]); } } - }.run(this, 10, INNER_LOOP); + }.run(getClass(), testInfo.getDisplayName(), 10, INNER_LOOP); } /* * Add KEYS_PER_NODE keys to a preference node and then remove them one at a time. */ - public void testRemoveStringKeys() { + @Test + public void testRemoveStringKeys(TestInfo testInfo) throws Exception { // gather the key/value pairs before so we don't time it final String qualifier = getUniqueString(); @@ -338,7 +337,7 @@ protected void test() { prefs.remove(key); } } - }.run(this, 50, 1); + }.run(getClass(), testInfo.getDisplayName(), 50, 1); } private String getUniqueString() { diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java index c9531478509..c5ac362cc2b 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/StartupTest.java @@ -13,15 +13,46 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; +import org.eclipse.core.tests.harness.session.ExecuteInHost; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.Performance; import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class StartupTest { + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS).create(); + @Test - public void testApplicationStartup() { + @Order(0) + public void warmup() { + testApplicationStartup(true); + } + + @Test + @ExecuteInHost + @Order(1) + public void activateRuntimePlugins() { + sessionTestExtension.setSystemProperty("eclipse.activateRuntimePlugins", "false"); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(2) + public void runMeasurements() { + testApplicationStartup(false); + } + + private void testApplicationStartup(boolean warmup) { PerformanceMeter meter = Performance.getDefault() .createPerformanceMeter(getClass().getName() + '.' + StartupTest.class.getName()); try { @@ -29,9 +60,7 @@ public void testApplicationStartup() { // tag for showing in the performance fingerprint graph Performance performance = Performance.getDefault(); performance.tagAsGlobalSummary(meter, "Core Headless Startup", Dimension.ELAPSED_PROCESS); - String reportOption = System.getProperty("eclipseTest.ReportResults"); - boolean bReport = (reportOption == null) ? true : !("false".equalsIgnoreCase(reportOption)); - if (bReport) { + if (!warmup) { meter.commit(); } Performance.getDefault().assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -100, 5); @@ -39,4 +68,5 @@ public void testApplicationStartup() { meter.dispose(); } } + } diff --git a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java index 50a6104ac88..755ef9716bd 100644 --- a/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java +++ b/runtime/tests/org.eclipse.core.tests.runtime/src/org/eclipse/core/tests/runtime/perf/UIStartupTest.java @@ -13,22 +13,48 @@ *******************************************************************************/ package org.eclipse.core.tests.runtime.perf; +import org.eclipse.core.tests.harness.session.PerformanceSessionTest; +import org.eclipse.core.tests.harness.session.SessionTestExtension; +import org.eclipse.core.tests.runtime.RuntimeTestsPlugin; import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.Performance; import org.eclipse.test.performance.PerformanceMeter; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.RegisterExtension; +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class UIStartupTest { + @RegisterExtension + static final SessionTestExtension sessionTestExtension = SessionTestExtension + .forPlugin(RuntimeTestsPlugin.PI_RUNTIME_TESTS) + .withApplicationId(SessionTestExtension.UI_TEST_APPLICATION).create(); + @Test - public void testUIApplicationStartup() { + @Order(0) + public void warmup() { + testUIApplicationStartup(true); + } + + @PerformanceSessionTest(repetitions = 5) + @Order(1) + public void runMeasurements() { + testUIApplicationStartup(false); + } + + public void testUIApplicationStartup(boolean warmup) { PerformanceMeter meter = Performance.getDefault() .createPerformanceMeter(getClass().getName() + '.' + UIStartupTest.class.getName()); try { meter.stop(); Performance performance = Performance.getDefault(); performance.tagAsGlobalSummary(meter, "Core UI Startup", Dimension.ELAPSED_PROCESS); - meter.commit(); + if (!warmup) { + meter.commit(); + } performance.assertPerformanceInRelativeBand(meter, Dimension.ELAPSED_PROCESS, -50, 5); } finally { meter.dispose();