[GR-70210] Outlier filtering of polybench wasm benchmarks.

PullRequest: graal/22296

Author: farquet

truffle/mx.truffle/mx_polybench/model.py

@@ -501,6 +501,8 @@ def _infer_guest_vm_info(self, benchmarks, bm_suite_args) -> Tuple[str, str]:
             guest_vm = "graal-js"
         elif "python" in resolved_benchmark.suite.languages:
             guest_vm = "graalpython"
+        elif "wasm" in resolved_benchmark.suite.languages:
+            guest_vm = "wasm"
         else:
             guest_vm = "none"
         if "--engine.Compilation=false" in self.runArgs(

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/Config.java

@@ -104,6 +104,7 @@ public void parseBenchSpecificDefaults(Value benchmark) throws InvalidObjectExce
     private void parseBenchSpecificSummary(Value benchmark) throws InvalidObjectException {
         if (!benchmark.hasMember("summary")) {
             // No 'summary' member provided in the benchmark
+            summary = parseFallbackBenchSpecificSummary(benchmark);
             return;
         }
         Value summaryMember = benchmark.getMember("summary");
@@ -131,6 +132,18 @@ private void parseBenchSpecificSummary(Value benchmark) throws InvalidObjectExce
         }
     }
 
+    private static Summary parseFallbackBenchSpecificSummary(Value benchmark) {
+        if (benchmark.hasMember(OutlierRemovalAverageSummary.class.getSimpleName())) {
+            double lowerThreshold = benchmark.getMember(OutlierRemovalAverageSummary.class.getSimpleName() + "LowerThreshold").execute().asDouble();
+            double upperThreshold = benchmark.getMember(OutlierRemovalAverageSummary.class.getSimpleName() + "UpperThreshold").execute().asDouble();
+            return new OutlierRemovalAverageSummary(lowerThreshold, upperThreshold);
+        } else if (benchmark.hasMember(AverageSummary.class.getSimpleName())) {
+            return new AverageSummary();
+        }
+        // defaulting to averaging if the aggregation strategy is not specified
+        return new AverageSummary();
+    }
+
     @Override
     public String toString() {
         String config = "metric:            " + metric.name() + " (" + metric.unit() + ")" + "\n" +

truffle/src/org.graalvm.polybench/src/org/graalvm/polybench/PolyBenchLauncher.java

@@ -394,6 +394,9 @@ private EvalResult evalSource(Context context) {
             } finally {
                 config.metric.afterLoad(config);
             }
+            if (language.equals("wasm")) {
+                result = result.newInstance();
+            }
             return new EvalResult(language, source.getName(), source.hasBytes(), source.getLength(), result);
         }
     }
@@ -475,6 +478,9 @@ private void runHarness(Context.Builder contextBuilder, boolean evalSourceOnly,
             log("::: Bench specific options :::");
             if (evalResult.value instanceof Value) {
                 Value value = (Value) evalResult.value;
+                if (evalResult.languageId.equals("wasm")) {
+                    value = value.getMember("exports");
+                }
                 config.parseBenchSpecificDefaults(value);
                 config.metric.parseBenchSpecificOptions(value);
             }
@@ -556,7 +562,7 @@ private Workload lookup(Context context, String languageId, Object evalSource, S
         // language-specific lookup
         switch (languageId) {
             case "wasm":
-                result = evalSourceValue.newInstance().getMember("exports").getMember(memberName);
+                result = evalSourceValue.getMember("exports").getMember(memberName);
                 break;
             case "java":
                 // Espresso doesn't provide methods as executable values.

wasm/benchmarks/interpreter/README.md

@@ -1,15 +1,9 @@
 ## Benchmark generation
 
-The `wat` files in this folder were generated from the provided C sources (which are included for reproducibility). To re-generate the `wat` files, one has to use the `clang` compiler contained in the `wasi-sdk` and `wasm2wat` from WABT. Follow the instructions in [the WASM README.](../../README.md)
-
-And then proceed by running the following commands:
+The `wat` files in this folder were generated from the provided C sources (which are included for reproducibility). To re-generate the `wat` files, you can use the provided `regenerate_wat.sh` script. First, ensure that `WASI_SDK` and `WABT_DIR` are set in your environment, then run:
 ```
-$WASI_SDK/bin/clang -Wl,--export="run" -O3 -o richards.wasm richards.c
-$WABT_DIR/wasm2wat -o richards.wat richards.wasm
+./regenerate_wat.sh
 ```
+This script uses `clang` from `WASI_SDK` and `wasm2wat` from `WABT_DIR` to regenerate all `.wat` files from the C sources.
 
-In the case of `sieve`, an additional linker flag must be passed to clang, since the default stack size (64kb) is too small.
-```
-$WASI_SDK/bin/clang -Wl,-export="run" -Wl,-z,stack-size=4194304 -O3 -o sieve.wasm sieve.c
-$WABT_DIR/wasm2wat -o sieve.wat sieve.wasm
-```
+Follow the instructions in [the WASM README.](../../README.md)

wasm/benchmarks/interpreter/deltablue.c

@@ -1099,6 +1099,17 @@ void ProjectionTest(int n)
   List_Destroy(dests);
 }
 
+void OutlierRemovalAverageSummary() {
+}
+
+double OutlierRemovalAverageSummaryLowerThreshold() {
+  return 0.0;
+}
+
+double OutlierRemovalAverageSummaryUpperThreshold() {
+  return 0.5;
+}
+
 int run()
 {
   int n = 1000;