Merge branch 'main' into mem_overh

Author: Manciukic

.buildkite/pipeline_perf.py

@@ -65,6 +65,11 @@
         "tests": "integration_tests/performance/test_boottime.py::test_boottime",
         "devtool_opts": "-c 1-10 -m 0",
     },
+    "process-startup": {
+        "label": "process-startup",
+        "tests": "integration_tests/performance/test_process_startup_time.py",
+        "devtool_opts": "-c 1-10 -m 0",
+    },
     "jailer": {
         "label": "jailer",
         "tests": "integration_tests/performance/test_jailer.py",

tests/host_tools/cargo_build.py

@@ -69,7 +69,12 @@ def get_example(name, *args, package="firecracker", **kwargs):
     return get_binary(package, *args, **kwargs, example=name)
 
 
-def run_seccompiler_bin(bpf_path, json_path=defs.SECCOMP_JSON_DIR, basic=False):
+def run_seccompiler_bin(
+    bpf_path,
+    json_path=defs.SECCOMP_JSON_DIR,
+    basic=False,
+    binary_dir=DEFAULT_BINARY_DIR,
+):
     """
     Run seccompiler-bin.
 
@@ -85,7 +90,7 @@ def run_seccompiler_bin(bpf_path, json_path=defs.SECCOMP_JSON_DIR, basic=False):
     if basic:
         seccompiler_args += " --basic"
 
-    seccompiler = get_binary("seccompiler-bin")
+    seccompiler = get_binary("seccompiler-bin", binary_dir=binary_dir)
     utils.check_output(f"{seccompiler} {seccompiler_args}")
 
 

tests/integration_tests/performance/test_process_startup_time.py

@@ -5,57 +5,63 @@
 import os
 import time
 
+import pytest
+
 from host_tools.cargo_build import run_seccompiler_bin
 
+ITERATIONS = 100
+
 
+@pytest.mark.nonci
 def test_startup_time_new_pid_ns(
     microvm_factory, guest_kernel_linux_5_10, rootfs, metrics
 ):
     """
     Check startup time when jailer is spawned in a new PID namespace.
     """
-    for _ in range(10):
+    for _ in range(ITERATIONS):
         microvm = microvm_factory.build(guest_kernel_linux_5_10, rootfs)
         microvm.jailer.new_pid_ns = True
         _test_startup_time(microvm, metrics, "new_pid_ns")
 
 
+@pytest.mark.nonci
 def test_startup_time_daemonize(
     microvm_factory, guest_kernel_linux_5_10, rootfs, metrics
 ):
     """
     Check startup time when jailer detaches Firecracker from the controlling terminal.
     """
-    for _ in range(10):
+    for _ in range(ITERATIONS):
         microvm = microvm_factory.build(guest_kernel_linux_5_10, rootfs)
         _test_startup_time(microvm, metrics, "daemonize")
 
 
+@pytest.mark.nonci
 def test_startup_time_custom_seccomp(
     microvm_factory, guest_kernel_linux_5_10, rootfs, metrics
 ):
     """
     Check the startup time when using custom seccomp filters.
     """
-    for _ in range(10):
+    for _ in range(ITERATIONS):
         microvm = microvm_factory.build(guest_kernel_linux_5_10, rootfs)
         _custom_filter_setup(microvm)
         _test_startup_time(microvm, metrics, "custom_seccomp")
 
 
 def _test_startup_time(microvm, metrics, test_suffix: str):
+    test_start_time = time.time()
     microvm.spawn()
     microvm.basic_config(vcpu_count=2, mem_size_mib=1024)
     metrics.set_dimensions(
         {**microvm.dimensions, "performance_test": f"test_startup_time_{test_suffix}"}
     )
-    test_start_time = time.time()
     microvm.start()
-    time.sleep(0.4)
+    datapoints = microvm.get_all_metrics()
 
-    # The metrics should be at index 1.
+    # The metrics should be at index 0.
     # Since metrics are flushed at InstanceStart, the first line will suffice.
-    datapoints = microvm.get_all_metrics()
     test_end_time = time.time()
     fc_metrics = datapoints[0]
     startup_time_us = fc_metrics["api_server"]["process_startup_time_us"]
@@ -68,20 +74,21 @@ def _test_startup_time(microvm, metrics, test_suffix: str):
     )
 
     assert cpu_startup_time_us > 0
-    # Check that startup time is not a huge value
+    # Check that startup time is not a huge value (overflow)
     # This is to catch issues like the ones introduced in PR
     # https://github.com/firecracker-microvm/firecracker/pull/4305
     test_time_delta_us = (test_end_time - test_start_time) * 1000 * 1000
     assert startup_time_us < test_time_delta_us
     assert cpu_startup_time_us < test_time_delta_us
 
-    metrics.put_metric("startup_time", cpu_startup_time_us, unit="Microseconds")
+    metrics.put_metric("startup_cpu_time", cpu_startup_time_us, unit="Microseconds")
+    metrics.put_metric("startup_time", startup_time_us, unit="Microseconds")
 
 
 def _custom_filter_setup(test_microvm):
     bpf_path = os.path.join(test_microvm.path, "bpf.out")
 
-    run_seccompiler_bin(bpf_path)
+    run_seccompiler_bin(bpf_path, binary_dir=test_microvm.fc_binary_path.parent)
 
     test_microvm.create_jailed_resource(bpf_path)
     test_microvm.jailer.extra_args.update({"seccomp-filter": "bpf.out"})