From f66dad519486686228b1bf352ad1da7059ef80c9 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Fri, 23 Jan 2026 13:46:32 -0500 Subject: [PATCH 1/4] feat: add boot time optimizations for faster VM startup This PR adds three optimizations to reduce VM boot time: 1. **Device polling instead of fixed sleep**: Replace the 500ms hardcoded sleep with a polling mechanism that waits for /dev/vda and /dev/vdb with 10ms intervals and 2s timeout. Typically completes in 10-50ms. 2. **Skip kernel headers via API parameter**: Add `skip_kernel_headers` boolean to CreateInstanceRequest. When true, skips the ~2s kernel headers extraction. Safe for workloads that don't need DKMS. 3. **Skip guest-agent via API parameter**: Add `skip_guest_agent` boolean to CreateInstanceRequest. When true, skips guest-agent copy (disables exec/stat API). Also includes lazy copy optimization that skips if the agent already exists in the overlay. New API usage: ```json { "name": "my-instance", "image": "node:20", "skip_kernel_headers": true, "skip_guest_agent": false } ``` Estimated savings: - Device polling: 400-490ms - Skip kernel headers: ~2s (when tarball present) - Skip guest-agent: 50-100ms --- cmd/api/api/instances.go | 2 + lib/instances/configdisk.go | 4 + lib/instances/create.go | 2 + lib/instances/types.go | 6 + lib/oapi/oapi.go | 305 +++++++++++++++++++----------------- lib/system/init/main.go | 16 +- lib/system/init/mount.go | 44 +++++- lib/vmconfig/config.go | 4 + openapi.yaml | 17 ++ 9 files changed, 246 insertions(+), 154 deletions(-) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e0cd9c7..4b6214e 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -229,6 +229,8 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Volumes: volumes, Hypervisor: hvType, GPU: gpuConfig, + SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, + SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index cc8b49e..7d3f73c 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -97,6 +97,10 @@ func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInf cfg.InitMode = "systemd" } + // Boot optimizations + cfg.SkipKernelHeaders = inst.SkipKernelHeaders + cfg.SkipGuestAgent = inst.SkipGuestAgent + return cfg } diff --git a/lib/instances/create.go b/lib/instances/create.go index d165f3b..904e589 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -335,6 +335,8 @@ func (m *manager) createInstance( Devices: resolvedDeviceIDs, GPUProfile: gpuProfile, GPUMdevUUID: gpuMdevUUID, + SkipKernelHeaders: req.SkipKernelHeaders, + SkipGuestAgent: req.SkipGuestAgent, } // 12. Ensure directories diff --git a/lib/instances/types.go b/lib/instances/types.go index b27f6e2..8c3b468 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -80,6 +80,10 @@ type StoredMetadata struct { // GPU configuration (vGPU mode) GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUMdevUUID string // mdev device UUID + + // Boot optimizations + SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) + SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) } // Instance represents a virtual machine instance with derived runtime state @@ -120,6 +124,8 @@ type CreateInstanceRequest struct { Volumes []VolumeAttachment // Volumes to attach at creation time Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) GPU *GPUConfig // Optional: vGPU configuration + SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) + SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 90d86f7..e047fe2 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -276,6 +276,17 @@ type CreateInstanceRequest struct { // Size Base memory size (human-readable format like "1GB", "512MB", "2G") Size *string `json:"size,omitempty"` + // SkipGuestAgent Skip guest-agent installation during boot. + // When true, the exec and stat APIs will not work for this instance. + // The instance will still run, but remote command execution will be unavailable. + SkipGuestAgent *bool `json:"skip_guest_agent,omitempty"` + + // SkipKernelHeaders Skip kernel headers installation during boot for faster startup. + // When true, DKMS (Dynamic Kernel Module Support) will not work, + // preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). + // Recommended for workloads that don't need kernel module compilation. + SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + // Vcpus Number of virtual CPUs Vcpus *int `json:"vcpus,omitempty"` @@ -9961,151 +9972,155 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XITubboq6j6nl3HOdt2nA+Y4FNTt0ICTPYhkEtI9j17wjVyt2xr0i31SGoHQ/F3", - "HmAecZ7klpak/rLa7gAx5MCuXTUmrc+lpfWttT4EIU9SzghTMhh+CGQ4IwmGn4dK4XB2yeMsIa/I7xmR", - "Sv85FTwlQlECjRKeMTVKsZrpf0VEhoKminIWDIMzrGboZkYEQXMYBckZz+IIjQmCfiQKugF5h5M0JsEw", - "2E6Y2o6wwkE3UItU/0kqQdk0+NgNBMERZ/HCTDPBWayC4QTHknRr057qoRGWSHfpQZ98vDHnMcEs+Agj", - "/p5RQaJg+Gt5G2/yxnz8GwmVnvxwjmmMxzE5JnMakmUwhJkQhKlRJOiciGVQHJnv8QKNecYiZNqhDsvi", - "GNEJYpyRrQow2JxGVENCN9FTB0MlMuKBTARrGtHIcwJHJ8h8RifHqDMj76qT7P40Pgiah2Q4IcuD/pIl", - "mPU0cPWy3PjQtjz2833fyJQnSTaaCp6lyyOfvDw9vUDwEbEsGRNRHvFgNx+PMkWmROgB05COcBQJIqV/", - "/+5jeW2DwWAwxLvDwaA/8K1yTljERSNIzWc/SHcGEVkxZCuQ2vGXQPri8uT45BAdcZFygaHv0kw1xC6D", - "p7yvMtpUT8WH/48zGkcerOd6YYpEI6yWNwWdkG1DOUOKJkQqnKRBN5hwkehOQYQV6ekvbVA9FASvmU63", - "aDXZMtJnBqajRDaN7pogylBC45hKEnIWyfIclKmH+82bKaEuEYJ7aMUT/WeUECnxlKCOJmCaijIkFVaZ", - "RFSiCaYxibbagMyHw2Yzv/ExohFhik5o9aYFY92gh8fhzu6e9xYneEpGEZ1anlAd/hj+jvgE6XEUgtb+", - "jWiUX7TbB0wpyGR5vqdARGESQSZEEBZ+9nSp4HPCMDPE/t9g3uB/bRfMcttyym0A5lnR/GM3+D0jGRml", - "XFKzwiUaYr9oNAJQI+jhXzN8WnXWJYySCovV9wNafIGbaNbXCjbnpmmdMgHhscNUbnYjAXoyJ0z5qBBT", - "9kN1x8/5FMWUEWRbWPhOuEB6gp9jPt0KvszeukEB0uULrdf9CQTJ/KFhNP2tGxCWJRqYMZ+WoTkjWKgx", - "qQCzgUHYgYrVNYL/rHIlqmcwxpKMVlOFM8oYiZBuaS+raYkyCXLg0vbhZlxTNZoTIb33CJb1X1Qh26Jx", - "qJiH1xMak9EMy5lZMY4iuIM4PqvsxCMLVYRLnGrC5gYEHi2R4uj8l8PdBw+RncADQ8kzEZoVLO+k1FsP", - "b9oihcUYx7EXN5rR7fZ8dxlD/Bhwnl+MJn6SY6BDTEO9AnuaevhukGZyZn4BPdarAn6myYBGr1j/fuPZ", - "9BEQCSODN2okfgnrZWoOG01jrmG6QBmjv2cV8bWPTrQkrpAm/jQiURdh+KDJMM4U700JI0LTKTQRPEFq", - "RlBJxEQd0p/2u+hKS109LWP28G5vMOgNroKqkBjv96ZppkGBlSJCL/D//Yp77w97/xr0Hr0pfo76vTd/", - "/zcfArSVezU66XXafXbc3e8it9iyMFxf6GpBeYWs6aMi5vhO9N2/7ekdnSwzeLP+iIfXRPQp347pWGCx", - "2GZTyt4NY6yIVNXdrG67dn+wthUbY1O99VturSb6A7p1Yn5DRKgpZUw0gsiuJpZUyS7CWnsEIoM0N/tP", - "FGKmcdYwdi4QYRG6oWqGMLSrQiBZ9HBKe9QsNegGCX73nLCpVt8f7i3ho0bGjv3Re/Mf7k9b/9uLkiKL", - "iQcZX/FMUTZF8Nlw3xmVqFgDVSRZy24ddLMYRKyEshPTbSdfCRYCL/yn5ha36vSk0sSn8fjMBfLs79gp", - "2BJZpQ0YAgbzCez32dnFtr6SKZZSzQTPprPyqfzq6MGbEiwapAG3yW4QUXk9onw0Tn1rovIanWy/RJpa", - "oZgmVBXUaWcwOH28La8C/Y8H7h9bfXRs7CqwfL15LizRlDMsCLDuCHGGjs4uEI5jHlplaKIlrAmdZoJE", - "/Zo2DKP7sIWw+Wfw4SdsTgVniZaF5lhQfXkqOv6H4MXL4yejJy8ug6E+ySgLrcJ89vLV62AY7A0Gg8DH", - "6vRJrEHGZ2cXR7Bj3X7GVRpn05Gk70nFOhXsPXsc1Bd+mO8XJSThwsijdgzUmVXJgWHXKKbXBF3p8cyh", - "7TyrE+pdmGoJaLNFSsScSp+e+Uv+TZ93Jkn5bprLUEUJScSciPys4fD7JV4fxjyLeqUpu8HvJAG0Lhbq", - "aeTX9VpxgTXkHccpZaSRvne/FZp8w8V1zHHU2/nCJJkRpcde3uIL86F6mBYBSH7+QXdJzmfRDY3UbBTx", - "G6aX7KE99gvKG+cE6J3eCY7/+uPPy9NCANl5Nk4tNdrZffCZ1KhGf/TQXuUi30iW+rdxkfo3cXn61x9/", - "up183U0QpvEzqhAdo69Xt/LPGVEzIkpcyR2w/pORDqE7cvhSmr5iACjbz5cIJ58TEeOFhxDuDDyU8J+C", - "Krhfth/SHA3pzmvIoB7NMa9lQjjwU0LPojxreqzvt6XLbVaSL2Rn99T+3G1Lm+dh6pQpu6Td+nJegBFc", - "i/BzKlSGY40nFTbntYkbb4tHLDDOnLJ4Ys8/xwesqibUtuKZGRlcL8vCil8iM1S+WSJb43nyGTZzLS/M", - "pOJJybyJOjUFjlZVveqJzXnci7DCQI9bMg2z3GWjfbIwQ5lDaULN0XTssQpoDKQMTekUjxeqKuDsDJaP", - "3g9oN74P1E0OLYMeJBop7vHTOGw5OdZwdG3b2A3B/TVSfDSfUM/IOaUqNFYqUVjznlmk1UP00pBab1oX", - "3cyopm0SOSAAQ7s8LQve/SvWQ3pxQ3ScT5APmw+pWTpYJ2CIDhelRVAwNKHxYgthdHnaR6/z1f67RAwr", - "OifOwzfDEo0JYSgDnkgimB/8luUFZFJrSFTVu1uZ3TgDt0C/4PZbH2kBLsEM3dA4BvtEghUNwbgxprX9", - "gFHZHJSeSRMAVoh5V6yMWdarWif5q90vr8iUSiVqzhfUefX0aG9v71GdSO8+6A12ejsPXu8MhgP9/3+1", - "99N8eX+nb6zDKr2w5qIyRTm6ODnetRyhOo96v48fHbx7h9Wjh/RGPnqfjMX0tz28EY+onzwdF3Yu1Mkk", - "ET1H+jRW+axbJSNSg/Xqk41St3LGOjP4KvZjdvdat7wL963PdWEN57d3sNaJ4FrnR2lzS/vRf9XyQYH5", - "JYXM2hhD6rWmHlN5/VgQfK1FeQ9/1exZjgzf8RsYMi28jheIvNNyLYmQ4FxNpFHSqmLKzv5P+wd7D/cP", - "BgOPr3QZiXlIR6HmKq0WoDXDGC+0cqr7oA5I1xEax3xcRd4Hew8Pfho82tltuw4jm7aDQy5FuV6oYyHy", - "dxcB475UFrW7+9PDvb29wcOHu/utVmUFvFaLcsJgRXT4ae+n/Z2D3f1WUPDJ+k+c77rui4s8SHqYpjE1", - "mk1PpiSkExoi8H4j3QF1EmBLJBezq3dyjKORsGKglx8oTGMPGEqmFjOZbYk6mqcnWaxoGhPzDQ6klaQL", - "Oz+GkXxmOcoYEaPctX+LkazHf605wu0lbwIiSkTG2XRq3CoF6E6pBMmiEIgoiaOhuaFr6RycZrGwN014", - "YPfQEhue8xsiejGZk7iMBIYd6cUmXBCU44k5tMquKJvjmEYjytLMixKNoHyaCZAvzaAIj3mmQJY0B1ae", - "BPwUoCNMNLlu5yYrDINLUz87u7ittSUVfEJjzzbmejD71bJ0Z4d4vj847+38HzA+vGTxwtAByhD0SXhE", - "+rUwLWjfentnTWvKY+RQeXVLe8KumccmlWu7DiISqZnWSDFDY4IsmzSWNLBTFpMUBP6Rj2BOBE7IOJtM", - "iBglHk3rqf6OTAOj+FOGTh9XiaYmzm3FrbPK4YC8NcEhZdOt1tD3aHK1bXRL0HzjP65XxPiSm1y3+qiE", - "bWO9t330Io9KRM/OLiTKZ+l7VLyWXpKz2UJq5cSMaCIxKCtrZoCcrcnwWdHR6rAeYpx4CZC7CKgzn6YZ", - "XMPzV72Tl5fbSUTm3cqa9MebGY+JXvdWSbaaOwdu4dKp2LnnTSKyQQzZ9gKVYJXf4NZAKt1XD3QUVzge", - "yZgrz2pe648IPqLO5VPjuNMr6KK0cpT67yUoVPD7offGaIrUNO05TFjXtSsXfK3ZIzFsq7y9yqS+q/IL", - "wbGJYa7icxEL5A6eX1cPml+vvb12EN+8J87VUeOciUd3OTo9NppZyJnClBGBEqKwjZguuRPBqx10g54W", - "BiJMEs4Qn0z+c7WDscF2k6PLKu3/aCns8k40/4bQIk3k4jmJUIIZnRCpbGhRZWY5w7sPHg5NUGNEJvsP", - "Hvb7fb9ZXYlFyqkvpuxJ/q3dUWwbp1SvGLMvZ593DnfgOG2zlw/B2eHrX4JhsJ1JsR3zEMfbckzZsPTv", - "/J/FB/hh/jmmzOtwbRUHSydL8a+V4001zzJ/H+qdMBLmCMlBSlxrm/Rz8hcaNWP6nkTIG4ai8BRp+Rsw", - "7vPiTT4jcrR4SKBKEaNlN0GL6FH6frW67QQjaGPnzJiicRFYu6xof1JotFwZabYUZZYSlseWxbH5FXI2", - "17fCF2hWIeDu29Jh3HBxTdl0FFEPdv7TfEQRFSRU4Mdff4eCbZym61HRL/zlNK1t0KwNmfFwl69OyT/F", - "4Fqd/eX0H7//X3n20287vz+/vPzv+bN/HL+g/30Zn738LD//6miprxrytNKnBlbGSqhTW/Q4xSr0CD4z", - "LlUD1OwXpDhKdOc+OgIFbXjFeug5VUTgeIiuApzSvgVmP+TJVYA65B0OlemFOEN6KDQjOCJiS3c+M7EO", - "uvMHpwN+rI8RLRhOaIiEBXLuQ5fZOOIJpmzril0xOxZyG5HgtNG/IhTiVGWC6BPRsma8QGOBtbpp1ehi", - "8i76gNP049YVA02UvFNC7yDFQuWhlW4GOGi7KuMUss1JhOY4zoi0muwVy/kHqOZ6EIXFlKh+bkIEQ03N", - "MdMAFK+awYWq+JYPBl3POSLdTh9kTKUiDOVWCSoBeVHHRQYcDCrX/2BwsN7/mOPQCvQD7F5+VuiQssX9", - "MAgMUxtiPJopla5/Jwj0xtwR9Mvr12caDPq/58gNVMAiP2KjjOE0jSmRxqumYpBJbDDGVuDznJnTbbmh", - "16ax7hbL9ft4AhOj18/PkSIioczQ706owTmhod4f+HeolJlGRYrR4dHpk61+i3eRANt8/SvO8XW+w5ob", - "wRm3ljVM6FEYzTV8u+jkuKvFKXtDC0EL/KZPuUCxITDFvR6iC0mqUQxwVMbFY04yXhQWMkPVr4ItN2Ja", - "pxRD9CqX73C+lDzku0AGN2RxL2HYK/ZPjRjGqbs0ere6VnBXW/3FkjZw4WKFrNEbWHEzKVh9/T0QhzvP", - "Wd32eLu7XTZa6sn8qFGc/Z1LIHu31SVvGz5bjQQqRX7lEbRfN/T1UwJZ3Qk9O7uAcFEsR5LhVM64ag7O", - "wMi1QeQdlUouB462CidYDpytsicTErsiGutLhsCKjDGIjKhv44sHt37NWINvL7B2ZSjs58azWgHtjsJZ", - "GwmCLxS0ShvMn79sYOqdLKcSYuojBmU+5gLBPjmqtBtQTxDMoZR0ykiETs6Kp1aFwcMNX9vTo93+zsOD", - "/s5g0N8ZtDH/JDhcMffp4VH7yQe7RiEe4vEwjIZk8hnmJ4vYRuDA8Q1eSHTlRMKrwMigJeGzdG2t2NjK", - "tbccvPtpsbp1JrguGvc20bet6P2qN9Dn1dfPreWKB//6rIfSpC0bPofGrtfoNoZRgkKexRH7d4XG+uYZ", - "VYBEVmORRBUPy+GyXrBrxm9YdevGPqbv7+8ZEQt0eXpasaYKMrFvbFtsnKdp4znw9FbHsLtGvFu7mlKw", - "9SYCrOuUsMSBvng4ddn04+I6DNa1MAEV4p/XTUqZAbc++xV7qinvEZmPsswn6OhPLkLz4uLkuHLgGD/c", - "ORgcPOodjHce9vajwU4P7+w97O0+wIPJXvjTXkM2ivZhEp8e+VC9oc0R0QB4MISZIPZoqO9QHrowzhTK", - "Xwbpy3mkJUZUkkNN/C/opq+MSKpHAO4a6i/xIhdVV3Y+w/qiur4p/Gt1j/NZprQYBH3kLFNI/wuWrLdg", - "Rf3VQ5g7P0QvOPSxK+1qRlnTGUxzzKLxYrl5Xb/o2AgQQaTigkQwmSVgQ/Q0J1o52bNkriOJ/WloqY2U", - "giiwLaNQW/HenlbQDSzUg25gQBh0AwcZ/dPsEH7B4oNuYBfiDbJcDlNYGRlRhFrU/eq3CaQpouiphFFp", - "KYYDdTQCli9zKRJ8q42M5hdU9DxNGYv0VWsb47I6pOUMq9kJm/Blk8JtmKV1FDoDTqqRRkIGiYgwSiIX", - "O5VzTYuH4HqMJUFRRizkDF4JbAGOjVklxWoGFx06UjatBl0tTdiGhZk1rH4zAfPahm2kbel3br0WGcDK", - "KMMS4cLN1Uqzp3Lkp8jLAwsyzWIsUD2Oa8WS5SKJKbtuM7pcJGMe0xDpDnVRaMLjmN+M9Cf5M+xlq9Xu", - "dIdRYdGtiTZmcdaebw6kNm+xhZ/1LrdqHsJQyyHbpv82pKRro7x4I52easZnQp0uGH1XQvRqAPL+7qDJ", - "IdwwaMUVvBwm1yamuHz3Lcr6bryLYDvMX0d6zInG0lXj/kcQu2a6VaMFvGFJYBFc5f7Ohyr5wJ0+5AK8", - "q3AtBVq3iut2ZNj7dCGXMRpcoivSdblh/ZT7pGw2r9tj5ok/wFYraU3QOjUq3DK8KlbmBwePHu3tP3i0", - "2wo0Vs/ODTUNhtsmY41bwbYkYe0hcvXEdh8M4H+3WpQx1fiX1GCuqS6o8qj4kxf0ccX1KQJAa2JEfj9W", - "JK0sTtLFilaOcv+gFbRWSCyHFbGnlFuiQyYTAoLvyMCtVyym5pBstYYQpzikauHx2eMb8NGgvEktkLHF", - "6LXFekBqx0Z4orTeOSdCZuMikr/jJkf/gcCGWcOFg9aPVmQ2HsEIHnNvfVZoZ52aUU35LdRtno3jkivF", - "PkfLM1v5LPg3OTDRDZYVq4T+HSoSdUu5Q+rmK9OifWo0h+t5drTCMO8LxvVnQisff+04u0GZmxToXIf4", - "KjbWfAU1VwaPaRsDgYcreiJ8w/UeoBp9sHzw03qNxuXnZCvf61XenrVOurI8rWFEt19uyZFwm471BzKA", - "VnYNFnLF2N3KyfqQwph5ml5fJy4pc+39DDXJNW2QNCo1Rh2SpGrhAgGdkrd1O7PTYT6gF6e+sOt28OhL", - "BI9drIwW+x/ynr9s6XOTrLXxLZ1pY4iGX+o8rnvRjHpl3zNWvT61V1pSrcghuypzuEnhDbqTDY+aZvV4", - "7ltkC2/Sloub45LDunTh65TAhmAI89i3tLPSSprPxph5PzO1OpUup/ongsxqMuvjjYypTOuSvfqDV/Nc", - "RlBQjSyADGA1CHJtd1mlXu19OsXv8hlA8cQS1fKymH2Ucpw9ewxP4F65h4904oaAZdQz7Dz+vJzzDquW", - "D2NVEnrnSPBePEt/VlC0prtVQ85iju7qPPeadJEwE1QtzjVDsD5yggURh5lBQ+AUsAn4czE5xNx9/Aja", - "5sQjdD4jjAgaosOzE8CSBDM81Ud2eYpiOiHhIoyJDZlaMjHDi7+XRyc9E+vpYgTAY00VAMQlwzg8O4F3", - "+DZlbDDo7/YhzxtPCcMpDYbBXn8HMg1oMMAWtyGUHn5am46+h8DJTiLLcR+bJhq0MuVMGuDsDga1FMS4", - "eOu8/Zs0xgrDXlvLdibb+rLbZykSyEkCdvkfu8H+YOdW61n7PNk37QXDmZpxQd8TWOaDWwLhkyY9YUY5", - "dlnoiG1Y4Gww/LWKrb+++fimG8gsSbAWEQ24ClilXDaJMEQijBi5sW8sfuPjPjo3qgW8VS7qWBjNn0Sa", - "JGGksOhP3yMswhmdkytmKbF5ao4FBJQmSFNgE85XRTMztTl9c4WJVI95tKhBNx9uWw8H0kgVwLfOzZzn", - "TUobkjT7qKNJzyBD7s1LQRhmqnjtb/IyXJMFSgWZ0HfekDwIM/Ibjo/zby6bd5W2a3GXsjDOooIBVrMo", - "e596SRIK4hOy/3H+8gWCiwf5mKFZER0FGbQo02QTRRlwHsCU/hV7gsMZMhQVkv9cBTS6Coq8u1tA/TJJ", - "DFHr9YAk/wwJyc00XRr93O/roQy1H6JfP5hRhugqYGkyUvyasKvgYxeVPkypmmXj/NubK+bdcIOufl6B", - "FeoYTN5y71n0DkuX2twCzCLELebEC4RRcUhlWX5MGRaLpiTWPFMjV0Wh4bmPbVbEoj8cDLbW25TtVj18", - "rtJQSwMfl8j67hejaJaaL1O0UsEKTT+YfcsVGTq+AZL6GEcuxPgH71jDO6zQW+IK0N9KDtsfaPTRoG9M", - "THxAjbRDXnNH2lMscEIUERLm9aEFhEZQ/W/nAQIl1aiAVeTtlsBTlwTfLCH2fmPC+Dz1OuDC/gbwD+Yt", - "MmzAvI82NS+OTX63vIjNvUJHOCyHiF2/2PqMqG8B4wabIqUuEdBXxN/7gj/PiJWEC6DVqNk2mTvzo9/P", - "rQTBibSjmMZaCD6HNfXOCVMISpXIvv2vk88gOuptzKdvh8iAMLaFWqRN7ZIbDzVTtLCETubla97PPggP", - "Z5hNiUQdwz//+uNPV2zirz/+tMUm/vrjT7ju27Z0EgyXl0l5O0T/RUjawzGdE7cZqbdA5kQs0N7AJvCF", - "T57n5fKKXbFXRGWCyTzmQ+8LYGIGhHc/DPZDWUYkkgBCSPw4scEIxjbh0Q3cXTag3OiN7i6pSHYHpQ1o", - "ruhwADxblFFFcYx4pkyOJlgHBJUWCzF7DsqT180sS4a39fRFkXfKYG/PLPCWBMaUGfLcO1N5x4yJOufn", - "T7b6CMR9gxUQcAJ6QzGM1QT6P2jSeppkKEqVoACUDW0qZRZqNNIc2zabsNI0ZR1qNtMISJFKtMbqNvND", - "7G5hsvHDzZlvfDaUY5cJs9mI8un79VUhaqVTfrlzdri3DHOb5rUA2dfQJlHHZujLH+JWcsl+LaTfCAEu", - "pSDOqTDi5vnvxjScI84mMQ0V6rm12Mo0udZTRZD7Qg5e2VUj7PZVj5Ius4rtSsBRI9OoFffdDPeoVxS+", - "BRspIrkLXPvBSdahzjGVIdd9S9jSC3FaqmEsi3taxqJ1tp1j+HvOclYK5nmtKFTU992QlcdOnbE6b9gA", - "UTyuEcSvSAhrz1RLbx/uEzZf5Kfosn6vMAJ9W6g52JwUtGmDkA/N75NFKKqBTVPBWZ4Yswm9bOrMOzxo", - "O4Nn4+dEuFttFmqeRxbbMl1ROCPhtdmQTU6/SiI4cfnr714OMPk/b8H97fJ/sPsWimMBq1XK4ol9M3t3", - "umKl5OmG3Y8WwTxABvf+uKgKTiLUwXLBwq3vygO5Ec5QTyZ/j27SWRbHzhA/J0IVWVDL9HT7g5YPWsjJ", - "7ratlEUuXj3vERZyCOYwoGsUSFzSwy8rLZsDM1v5gSZt9CsAlUOMZmH0M87fRHcWFZ//tvvU5iP62+5T", - "k5Hob3uHReHnu0GWwaZI86al13uMfFp4pVWgAWkyqQnXSXt5q40IfDYH7G1EvnyBP6S+NlJfGVwrBb88", - "He8din7VovAb9hPkyOaDNnxy8Wffmci3WdOTxchS4ZqKLd4ml4Ay4DazqC17cf8C5GiOcWX629KGWlzI", - "ldKBQ92T465NGmtSveaRxRuyqLp1bFxKtPNu3px6mIzpNOOZLOemhBzBRBYF1SoE+L7JrwV7bpRgv2Es", - "HWySdWxcQP2B93ckOtcP1BBvW4NtjfDsWm1GeC5cNe2lZ7fCH9JzK+m5BK7V0nOe/u8uxWczyVeTnx2+", - "+QBu375+jxL0fXu2wayNu+TsrdC41gJqkfJyNe8vCuRv3NGfT755udTlkbqf4afcBJxHThIseE2zKPit", - "4cNgs7Rv8yLgfUaxZ+VyJ35hy7y9iPl0/cuLfCT3zMDz9OKKudoob81zyLcoR1SkOJIkJqFCNzMazuAZ", - "hv4bjG9eaeA0fZu/u9waomcQ3ll+CQqTdyQRFMco5Ezy2OSCfTtPkrfD5bf8l6en0Mm8wDCv9t8OkXu/", - "n98xqVuVn1XoXcRYKvTCPhbp6AMXPI5NofO3Gp6l/W3ZBxfFE9Ur5nt8wciNHZBO0NvSO4y3DQ8xHBI+", - "16f0lW5+tzmTtdmL4kgA4EzdGgIVrH2PMGx1bc8TjJ2BNyVNy+cgZhl3/Bqku1xcfJq/+66gMk7Ttuhr", - "lwlYPE+SFTiMOkWtDyRVxDP1d6kiIkw5M4vdTciNOjg0/1D42hTfqlQfMRmKfaCyT5u9oApMiUGX2Nj8", - "a54kgSmFkmBfouLPf1ZTH3BZH9MnU3o784Nn3OZVTJXYl57F1DiHzZANWSC8ytsr0+C7l1xcKvGvjIab", - "d0WUVkGhHASLxgs42yJH+/16EwAHWewM+J3dl/eOuG+Nd8Smdv/u70iBH9/5LQm5gHqQ0tVnuT/BWyWN", - "o3TdO1AQoii00HVa7+Xp6VbTpTElBRuvjPihDts4yu+ep0CNjPt3W0x5KJxvYJWxUF8I1aijO521Uj9n", - "zDM9+lJaS6h9IBdSkcQo7JMshodtELVu8wPgcm2HLqJKQpbjLpisSnn9r9iYTDQ/TInQc+vukDar0D18", - "au25wvn1PTN38NvQayHTJahyWDVBbanEukty6dOd8rycn7ykp6CoVmtLSNSJ6bUpNoPmEsX6x9ZKTdcU", - "nvjS2Q8+/WblpVV8r1oNzubI/D1QuJMaWXNll+4dWXtGypfF0R84aB9Z4+kqNs/TH1ze1oD6IRPfS5kY", - "HD35bjpTgUPguNJW+fLLv7a8y/YH8+NknbtQ4XB26VKEfxus1GYUXjeN2+C9uJR2TxExT3o3fyd5nvT5", - "nj7b0IBzWwDTSdnx6ecCJpn894bdXz7GpQzHW0W4bPRuuefy38zd2jTns2tw4dpleNyXa24wze0EkheX", - "VVtRLk6zUqF1RUegUpLrllf56ZZLN5nsfLmCWhQLyKvE9K9YXhbHZQdER2cXXVsEugtlps0ItqhKH/mr", - "F0mEBXEljK6Y4ijEcZjFWBGUl/Expbdkg1v3Vam01Z3dt2ISz0Hn9YtkXqfmPukYfpyA0ysXwgGMK1VQ", - "bowttcWUNxJZapnZLeJK3Q5+hOC1iCotAatN2n3TvI/OszTlQkmkbjjUsZTgy4cki2MeLYYo78eQKX1k", - "SZytWWPzz5MI6obovqeVXPylAVzPVJBeylMgHZF5smNhbMSj5Sz/DYn8c/no7sJj66JD97a1AUprqZ5H", - "dY8oT7xvc8Fr2Fp4uSFaZXz3FTrJiw+EmVQ8ceOeHKMOzhTvTQnTwC3y/KeCz2lULxf3jdR4OsXvaJIl", - "eYHUZ4+h3KQwoR5QeBgCjRxOkXchIZGEyI+tW9aDWi4FZc/i03Lefzki5qhpo0z5FWOmi+SE+oi1jOmQ", - "XHGOYiymZOu7eZlo71rxMPHkuPYs8R5Ge88d9hVyRsv47nYqbUtN8y5iu3Nzx2Yjuy+/HS2slL/tHj4v", - "nOdiZlNI+beFgoPNsYRNh5Jf3mOrnda25jWwmQH0iD6Eec5DHKOIzEnMU6h4aNoG3SATsa3fNtze1mpa", - "rBW54cHgYBB8fPPx/wcAAP//PPEgqyLOAAA=", + "H4sIAAAAAAAC/+x9a3MbOZLgX0HU7cRQOyRFPeyWudFxIUu2WzOWrbMs9e20fDRYBZJoVQHVAIoS2+Gv", + "/QP6J/YvuUACqBdRZMkP2lp7Y2OaVuGZyEzkC5nvgpAnKWeEKRkM3wUynJEEw89DpXA4u+RxlpBX5LeM", + "SKX/nAqeEqEogUYJz5gapVjN9L8iIkNBU0U5C4bBGVYzdDMjgqA5jILkjGdxhMYEQT8SBd2A3OIkjUkw", + "DLYTprYjrHDQDdQi1X+SSlA2Dd53A0FwxFm8MNNMcBarYDjBsSTd2rSnemiEJdJdetAnH2/MeUwwC97D", + "iL9lVJAoGP5S3sabvDEf/0pCpSc/nGMa43FMjsmchmQZDGEmBGFqFAk6J2IZFEfme7xAY56xCJl2qMOy", + "OEZ0ghhnZKsCDDanEdWQ0E301MFQiYx4IBPBmkY08pzA0Qkyn9HJMerMyG11kt0fxgdB85AMJ2R50J+y", + "BLOeBq5elhsf2pbHfr7vG5nyJMlGU8GzdHnkk5enpxcIPiKWJWMiyiMe7ObjUabIlAg9YBrSEY4iQaT0", + "7999LK9tMBgMhnh3OBj0B75VzgmLuGgEqfnsB+nOICIrhmwFUjv+EkhfXJ4cnxyiIy5SLjD0XZqphthl", + "8JT3VUab6qn48P9xRuPIg/VcL0yRaITV8qagE7JtKGdI0YRIhZM06AYTLhLdKYiwIj39pQ2qh4LgNdPp", + "Fq0mW0b6zMB0lMim0V0TRBlKaBxTSULOIlmegzL1cL95MyXUJUJwD694ov+MEiIlnhLU0QxMc1GGpMIq", + "k4hKNME0JtFWG5D5cNhs5lc+RjQiTNEJrVJaMNYNengc7uzueak4wVMyiujU3gnV4Y/h74hPkB5HIWjt", + "34hG+UW7fcCUgkyW53sKTBQmEWRCBGHhR0+XCj4nDDPD7P8D5g3+13ZxWW7bm3IbgHlWNH/fDX7LSEZG", + "KZfUrHCJh9gvGo0A1Ah6+NcMn1addQmjpMJiNX1Ai09AiWZ9rWBzbprWORMwHjtMhbIbGdCTOWHKx4WY", + "sh+qO37OpyimjCDbwsJ3wgXSE/wY8+lW8Gn21g0KkC4TtF73BzAk84eG0fS3bkBYlmhgxnxahuaMYKHG", + "pALMhgvCDlSsrhH8ZxWSqJ7BGEsyWs0VzihjJEK6pSVW0xJlEuTApe0DZVxTNZoTIb10BMv6F1XItmgc", + "Kubh9YTGZDTDcmZWjKMIaBDHZ5WdeGShinCJU83Y3IBwR0ukODr/6XD3wUNkJ/DAUPJMhGYFyzsp9dbD", + "m7ZIYTHGcezFjWZ0u/u9u4whfgw4zwmj6T7JMdAhpuFegT1NPXw3SDM5M7+AH+tVwX2m2YBGr1j/fuPZ", + "9BEwCSODN2okfgnrZWoOG01jrmG6QBmjv2UV8bWPTrQkrpBm/jQiURdh+KDZMM4U700JI0LzKTQRPEFq", + "RlBJxEQd0p/2u+hKS109LWP28G5vMOgNroKqkBjv96ZppkGBlSJCL/D//YJ7vx/2/j3oPXpT/Bz1e2/+", + "8R8+BGgr92p00uu0++w42u8it9iyMFxf6GpBeYWs6eMi5vhONO3f9fSOTpYveLP+iIfXRPQp347pWGCx", + "2GZTym6HMVZEqupuVrdduz9Y24qNsane+h23VhP9Ad06Mb8hItScMiYaQWRXM0uqZBdhrT0Ck0H6Nvsv", + "FGKmcdZc7FwgwiJ0Q9UMYWhXhUCy6OGU9qhZatANEnz7nLCpVt8f7i3ho0bGjv3Re/Of7k9b/9uLkiKL", + "iQcZX/FMUTZF8NncvjMqUbEGqkiy9rp10M1iELESyk5Mt518JVgIvPCfmlvcqtOTSjOfxuMzBOTZ37FT", + "sCWyShtcCBjMJ7DfZ2cX25okUyylmgmeTWflU/nF8YM3JVg0SANuk90govJ6RPlonPrWROU1Otl+iTS3", + "QjFNqCq4085gcPp4W14F+h8P3D+2+ujY2FVg+XrzXFimKWdYELi6I8QZOjq7QDiOeWiVoYmWsCZ0mgkS", + "9WvaMIzuwxbC5h9xDz9hcyo4S7QsNMeCauKp6Pjvghcvj5+Mnry4DIb6JKMstArz2ctXr4NhsDcYDALf", + "VadPYg0yPju7OIId6/YzrtI4m44k/Z1UrFPB3rPHQX3hh/l+UUISLow8asdAnVmVHZjrGsX0mqArPZ45", + "tJ1ndUa9C1MtAW22SImYU+nTM3/Kv+nzziQp06YhhipKSCLmRORnDYffL931YcyzqFeashv8RhJA62Kh", + "nkZ+Xa/VLbCGveM4pYw08vfu18KTb7i4jjmOejufmCUzovTYy1t8YT5UD9MiAMnPP+guyfksuqGRmo0i", + "fsP0kj28x35BeeOcAd3qneD4rz/+vDwtBJCdZ+PUcqOd3QcfyY1q/EcP7VUu8o1kqX8bF6l/E5enf/3x", + "p9vJl90EYRo/owrTMfp6dSs/z4iaEVG6ldwB6z8Z6RC6I4cvpekrBoCy/XyJcfI5ETFeeBjhzsDDCX8W", + "VAF92X5I32hId17DBvVo7vJaZoQDPyf0LMqzpseavi1fbrOSfCE7u6f2525b3iyvaTqaamFjhKe5AWOV", + "Z+P8mqYIevSghznGODbEG2V6ZDTmXPWv2M8zwhCcHRwwuSUh8CmtoaHDsxOJbmgcg7oDjGCZ91+x1yVW", + "YJpLpf9XZKyLxplCgiRcEa1rJnpsPUkGa4HGY4Iyhp3rpH/FylCxG6zjlQXLNRGMxKMZwRERsiVkTCdk", + "OzUCB7Y6wVIRYTh0llbhdfyv03PUOV4wnNAQ/cuMesqjLCboPEs1DW9Vode9Yqkgc8JA0NUCA7Xz8gni", + "merxSU8JQtwSExgsVxitXX/+7OzCeobkVv+KvSIasIRFWt/kArlbQiI1wwpFnP1dUyyJqsOW568B3U/L", + "3WAeplkVyrt1CL8Af4zez5wKleFYs6yKxOV1zxjHn0dCNX7FsqRsWVGOcFhVrfltNQUzMngBl+Vmv3Jg", + "BI5m5WCNE9RnY88NDmEmFU9KlnbUqdkSaNXqUGUecx73IqwwiAYt5Rez3GX/UbIwQ5lDaeKSo+nYY6DS", + "zJAyNKVTPF6oqqy9M1g+ej+g3fg+UDf5Vg16kGikuMdl6LDl5FjD0bVtY8IGT+xI8dF8Qj0j55dmYTyh", + "EoU1R65FWj1ELw2pJd8uuplRfc1K5IAAFHx5WtYB+1esByxniI7zCfJh8yE1ZwVDGQzR4aK0CAo2TzRe", + "bCGMLk/76HW+2r9LxLCic+KczTMs0ZgQhjIQz0gE8wM7LS8gk5qHUVXvbnmV8UtvgarL7bc+0rpEgi3f", + "1+idYEVDsLONaW0/4N8wB6Vn0gyAlW+dVrfEKk/gKzKlUomaHxB1Xj092tvbe1SXF3Yf9AY7vZ0Hr3cG", + "w4H+/3+3dxl+ete7b6zDKr+wlssyRzm6ODnetcJJdR71+z5+dHB7i9Wjh/RGPvo9GYvpr3t4I855P3s6", + "LkyuqJNJInqO9Wms8hlaS/bMBkPqB9tH7xQX4Dwyq64fs7vXuuXniCTwedGsD+fuvv46E1zrhyttbmk/", + "+q9aPigwv2QbsObukHoN+8dUXj8WBF9rrdJzv+rrWY7MveO3dWVajxovELnV4hmJkOBcTaSxF1TFlJ39", + "H/YP9h7uHwwGHrf9MhLzkI5Cfau0WsDLoxMU4wURCPqgDih6ERrHfFxF3gd7Dw9+GDza2W27DqMmtYND", + "LkW5XqhjIfIPF4zlvlQWtbv7w8O9vb3Bw4e7+61WZQW8VotywmBFdPhh74f9nYPd/VZQ8KmdT1wYRd0t", + "HHmQ9DBNY2qU7J5MSUgnNEQQiIF0B9RJ4FoiucZXpckxjkbCioHe+0BhGnvAULL6mclsS9TRd3qSxYqm", + "MTHf4EBaSbqw82MYyWchpowRMcqjTO4wkg0+WWsZc3vJm4CIEpFxNp0aD18BulMqQbIoBCJK4mhoKHQt", + "n4PTLBb2pgkP7B5aYsNzfkNELyZzEpeRwFxHerEJFwTleGIOrbIryuY4ptGIsjTzokQjKJ9mAuRLMyjC", + "Y54po6rDgZUnAZcZ6AgTza7beWwLG/XS1FrPvKPhLxV8QmPPNkBptV/tle5MYs/3B+e9nf8DdrCXLF4Y", + "PkCZUXQTHpF+LWIQ2rfe3lnTmvJwTVRe3dKectOExzyaa7sOIlbpDjFDY4LsNWmMumA2KSYpGPwjH8Oc", + "CJyQcTaZEDFKPJrWU/0dmQbGBkUZOn1cZZqaObcVt84qhwPy1gSHlE23WkPfo8nVttEtQfON/7heERPW", + "0BRFoI9K2DY2kKCPXuQBsujZ2YVEhTnJo+K1dNidzRZSKydmRBMURFlZMwPkbM2Gz4qOVof1MOPEy4Ac", + "IaDOfJpmQIbnr3onLy+3k4jMu5U1gQloxmOi171Vkq3mLpag8C5WXC7zJhHZIIZsS0AlWOUU3BpIJXr1", + "QEdxheORjLnyrOa1/ojgI+pcPjU+ZL2CLkorR6n/XoJCBb8feilGc6Smac9hwrquXSHwtWaPxFxb5e1V", + "JvWRyk8ExyacvorPRViaO3h+XT1ofr2Weu0gvnlPnNetdnMmHt3l6PTYaGYhZwpTRgRKiMI2eL/k2YYA", + "i6Ab9LQwEGGSgE108l+rfd0NtpscXVZp/0dLEcCfRfNviHLTTC6ekwglmNEJkcpGuVVmljO8++Dh0MTX", + "RmSy/+Bhv9/3e3iUWKSc+sIbn+Tf2h3FtvGP9oox+3L2cefwGXz4bfbyLjg7fP1TMAy2Mym2Yx7ieFuO", + "KRuW/p3/s/gAP8w/x5R5ff+tQrLpZCkUu3K8qb6zzN+HeieMhDlCcpAS19om/Tf5C42aMf2dRMgbEaXw", + "FGn5GzDu40KfPiKIuXjTokrBy2U3QYtAZvr7anXbCUbQxs6ZMUXjIsZ7WdH+oCh9uTLocSngMSUsD3OM", + "Y/Mr5GyuqcIX81hh4O7b0mHccHFN2XQUUQ92/mw+oogKEioIKVlPQ8E2TtP1qOgX/nKe1jZ+20ZveW6X", + "L87JP8TgWp395fSfv/1fefbDrzu/Pb+8/O/5s38ev6D/fRmfvfyokJPVgXtfNPpupU8NrIyVqLu26HGK", + "VegRfGZcqgao2S9IcZTozn10BAra8Ir10HOqiMDxEF0FOKV9C8x+yJOrAHXILQ6V6YU4Q3oo6zre0p3P", + "TNiN7vzO6YDv62NE1kcsLJDzcA6ZjSOeYMq2rtgVs2MhtxEJThv9K0IhTlUmiD4RLWvGCzQWOCx8w8Xk", + "XfQOp+n7rSsGmii5VULvIMVC5VG+bgY4aLsq4xSyzUmE5jjOiLSa7BXL7w9QzfUgCospUf3chAiGmppj", + "pgEoXjWDi2psw8Gg6zlHpNvpg4ypVISh3CpBJSAv6rgglYNBhfwPBgfr/Y85Dq1AP8Du5ReuDilb0IdB", + "YJjaMOPRTKl0ffgC8BtDI+in16/PNBj0f8+RG6iARX7ERhnDaRpTIo1XTcUgk9i4oK3A5zkzp9tyQ69N", + "Y90tbhGG8QQmRq+fnyNFREKZ4d+dUINzQkO9P/DvUCkzjYoUo8Oj0ydb/RZPdAG2+fpXnOPrfIc1N4Iz", + "bi1rmNCjMJpr+HbRyXFXi1OWQgtBC/ymT7lAsWEwBV0P0YUk1SgGOCrj4jEnGS8KC5nh6lfBlhsxrXOK", + "IXqVy3c4X0r++qBABjdkQZcwrA1sMU7dpdG71bWCu9rqL5a1gQsXK2SN3nAVN7OC1eTvgTjQPGd12+Pd", + "aLtstNST+VGjOPvPLoHs3VWXvGskdzUorRSEmAdzf9ko7A+JqXYn9OzsAiKXsRxJhlM546o5OAMj1waR", + "WyqVXI5jaxVOsBzDXb2eTHT2isDATxmNLTLGIDKivo1PHmf9JWMNvr4Y75VR2R8bWm0FtM8UWd3IEHxR", + "yVXeYP78aWOkP8tyKtHOPmZQvsdcINgHBzh3A+oJgjmUkk4ZidDJWfHqrzB4uOFre3q02995eNDfGQz6", + "O4M25p8EhyvmPj08aj/5YNcoxEM8HobRkEw+wvxkEdsIHDi+wQuJrpxIeBUYGbQkfJbI1oqNrVx7y3Hk", + "HxY2Xr8E1wWG3yUQvF2E94rn+OfVh/it5YoH//6oN/uk7TV8Do1dr9FdDKMEhTyLI/Z3hcaa8owqQCKr", + "sUiiihwHQKwX7JrxG1bdurGPafr9LSNigS5PTyvWVEEm9rl3i43zNG08B57e6Rh214h3a1dTCrbeRIB1", + "nROWbqBPHk5dNv24uA6DdS1MQIX453WTUmbArc9+xZ5qyntE5qMs8wk6+pOL0Ly4ODmuHDjGD3cOBgeP", + "egfjnYe9/Wiw08M7ew97uw/wYLIX/rDXkBilfZjEh0c+VCm0OSIaAA+GMBPEHg01DeWhC+NMofyRmibO", + "Iy0xopIcauJ/QTd9ZURSPQLcrqH+Ei9yUXVl5zOsCdX1TeFfq3uczzKlxSDoI2eZQvpfsGS9BSvqrx7C", + "0PwQveDQx660qy/Kms5gmmMWjRfLzev6RcdGgAgiFRckgsksAxuipznTytmeZXMdSexPw0ttpBREgW0Z", + "hdqK9/a0gm5goR50AwPCoBs4yOifZofwCxYfdAO7EG+Q5XKYwsrIiCLUou5Xv0sgTRFFTyWMSksxHKij", + "EbBMzKVI8K02MppfUNHzNCXP0qTWNsZldUjLGVazEzbhyyaFu1yW1lHoDDipRhoJyUwiwiiJXOxUfmta", + "PATXYywJijJiIWfwSmALcGzMKilWMyB06EjZtBp0tTRhmyvMrGH1mwmY1zZsI21Lv3PrtcgAVkYZlggX", + "bq5Wmj2VIz9HXh5YkGkWY4HqcVwrliwXSUzZdZvR5SIZ85iGSHeoi0ITHsf8ZqQ/yR9hL1utdqc7jAqL", + "bk20MYuz9nxzILV5iy38qHe5VfMQhloO2Tb9tyE7YhvlxRvp9FRffCbU6YLR2xKiVwOQ93cHTQ7hhkEr", + "ruDlMLk2McVl2rco66N4F8F2mD/U9ZgTjaWrdvsfQeya6VaNFvCGJYFFcJX7Ox+q5AN3+pAL8K7CtRRo", + "3Squ27Fh79OFXMZocImuyBznhvVz7pOy2bxuj5kn/gBbraQ1QevUqHDL8KpYmR8cPHq0t//g0W4r0Fg9", + "OzfUNBhum4w1bgXbkoS1N/HVE9t9MID/u9OijKnGv6QGc011QZX37R+8oPcryKcIAK2JETl9rMifWpyk", + "ixWtHOX+QStorZBYDitiTynNSYdMJgQE35GBW69YTM0h2WoNIU5xSNXC47PHN+CjQXmTWiBji9Fri/WA", + "1I6N8ERpvXNOhMzGRSR/x02O/hOBDbOGCwetH63IbDyCETzm3vqs0M46NaOa8luo2zwbxyVXin2OlidZ", + "81nwb3JgohssK1YJ/TtUJOqW0tjUzVemRfssfQ7X80R9hWHeF4zrT8pXPv7acXaD8m1SoHMd4quusWYS", + "1LcyeEzbGAg8t6Inwjdc7wGq8Qd7D35Yr9G4/Jxs5Xu9ytuz1vl/lqc1F9Hdl1tyJNylY/2BDKCVXYOF", + "XDF2t3KyPqQwZp6m19eJyw9eez9DTZ5XGySNSo1RhySpWrhAQKfkbd3N7HSYD+jFqU/suh08+hTBYxcr", + "o8X+h7znL1v63CRrbXxLZ9oYouGXOo/rXjSjXtn3jFWvT+2VllQr0hmvSmJvssmD7mTDo6ZZPZ77Donr", + "m7TlgnJcnmKXuX6dEtgQDGEe+5Z2VlpJ89kYM+9HZvmn0qX3/0CQWU1mfbyRMZVpXbJXf/BqnssICqqR", + "BZABrAZBru0uq9SrvU+n+DafARRPLFEtRZDZRynd3rPH8ATulXv4SCduCFhGPdnT448rf+CwavkwVtVD", + "cI4EL+FZ/rOCozXRVg05izm6q0suaNZFwkxQtTjXF4L1kRMsiDjMDBrCTQGbgD8Xk0PM3fv3oG1OPELn", + "M8KIoCE6PDsBLEkww1N9ZJenKKYTEi7CmNiQqSUTM7z4e3l00jOxnnlOHsihrAAgLhnG4dkJvMO32YuD", + "QX+3DykHeUoYTmkwDPb6O5BpQIMBtrgNofTw09p0NB3CTXYS2Rv3sWmiQStTzqQBzu5gUMuGjYu3ztu/", + "SmOsMNdra9nOJP5fdvssRQI5ScAu/3032B/s3Gk9a58n+6a9YDhTMy7o7wSW+eCOQPigSU+YUY5dQkRi", + "GxY4Gwx/qWLrL2/ev+kGMksSrEVEA64CVimXTSIMkQgjRm7sG4tf+biPzo1qAW+Vi5IqRvMnkWZJGCks", + "+tPfERbhjM7JFbOc2Dw1xwICShOkObAJ56uimZnanL4hYSLVYx4tatDNh9vWw4E0UgXwndOE53mT0oZ8", + "4T7uaNIzyJB781IQhpkqXvubvAzXZIFSQSb01huSB2FGfsPxcf7NJZav8nYt7lIWxllUXIDVhN7ep16S", + "hIL4hOx/nr98gYDwIDU4NCuioyCDFmWabeZJzvSx9a/YExzOkOGokPznKqDRVVCkgN4C7pdJYpharwcs", + "+UfIjW+m6dLox35fD2W4/RD98s6MMkRXAUuTkeLXhF0F77uo9GFK1Swb59/eXDHvhht09fMKrFDHYPKW", + "e8+id1giakMFmEWIW8yJFwij4pDKsvyYMiwWTfnUeaZGrqBHw3Mf26yIRX84GGyttynbrXruuUpDLQ28", + "X2Lru5+Mo1luvszRSrVTNP9g9i1XZPj4BljqYxy5EOPvd8eau8MKvaVbAfpbyWH7HY3eG/SNiYkPqLF2", + "SLHvWHuKBU6IgiSLv/hxHkIjqP638wCBkmpUwCrydkvgqUuCb5YQe7+xdkFeBQBwYX8D+AfzFhk2YN5H", + "m5oXxya/W15P6V6hIxyWQ8SuX2x9RtTXgHGDTbFSlwjoC+LvfcGfZ8RKwgXQatxsGzKrlnWiegifIDiR", + "dhTTWAvB57Cm3jlhCkHVHNm3/3XyGURHvY359O0QGRDGtmaQtKldcuOhvhQtLKGTefma97MPwsMZZlMi", + "Ucfcn3/98aere/LXH3/auid//fEnkPu2reIFw+UVe94O0b8ISXs4pnPiNiP1FsiciAXaG9hc0vDJ87xc", + "XrEr9oqoTDCZx3zofQFMzIDw7ofBfijLiEQSQAiJHyc2GMHYJjy6gaNlA8qNUnR3SUWyOyhtQN+KDgfA", + "s0UZVRTHiGfK5GiCdUBQabEQs+egPHndzLJkeFvPXxS5VQZ7e2aBd2QwpuKVh+5MESgzJuqcnz/Z6iMQ", + "9w1WQMAJ6A3FMFYT6H/nSet5kuEoVYYCUDa8qZRZqNFIc2zbbMJK05R1qNlMIyBFKtEaq9vMd7G7hcnG", + "DzdnvvHZUI5dJsxmI8qH79dXEKuVTvnpztnh3jLMbZrXAmRfQptEHZuhL3+IW8kl+6WQfiMMuJSCOOfC", + "iJvnvxvTcI44m8Q0VKjn1mKLJOVaTxVB7gs7eGVXjbDbVz1KunxVbFcCjhovjVqd6c3cHvXi1ne4RopI", + "7gLXvt8k61DnmMqQ674lbOmFOC2V05YFnZaxaJ1t5xj+nl85KwXzvGwZKkpNb8jKY6fOWP1u2ABTPK4x", + "xC/ICGvPVEtvH+4TNl/kp+iyfq8wAn1dqDnYnBS0aYOQD83vk0UoqoFNc8FZnhizCb1s6szPeNB2Bs/G", + "z4lwVG0Wap5HFtsyXVE4I+G12ZBNTr9KIjhx+es/vxxg8n/e4fa3y/9+3bdQHAtYrVIWT+yb2c+nK1aq", + "727Y/WgRzANkcO+PiwL1JEIdLBcs3PqmPJAbuRnqyeTvESWdZXHsDPFzIlSRBbXMT7ffafmghZzsqG2l", + "LHLx6nmPsJBDMIcBXaNA4pIeflpp2RyY2cp3NGmjXwGoHGI0C6Mfcf4murMoPv633ac2H9Hfdp+ajER/", + "2zssapB/HmQZbIo1b1p6vcfIp4VXWgUasCaTmnCdtJe32ojAZ3PA3kXkyxf4XeprI/WVwbVS8MvT8X5G", + "0c9mOf0yfoIc2XzQhk8u/uwbE/k2a3qyGFkqXFOxxdvkElCR3mYWtWUv7l+AHM0xrsx/W9pQC4JcKR04", + "1D057tqksSbVax5ZvCGLqlvHxqVEO+/mzamHyZhOM57Jcm5KyBFMZFFQrcKA75v8WlzPjRLsV4ylg01e", + "HRsXUL/j/WcSnesHapi3rcG2Rnh2rTYjPBeumvbSs1vhd+m5lfRcAtdq6TlP//c5xWczyReTnx2++QBu", + "375+ixL0fXu2wayNu+TsrfC41gJqkfJy9d1fFMjfuKM/n3zzcqnLI3U/w0+5CTiPnCRY3DXNouDXhg+D", + "zfK+zYuA9xnFnpXLnfiFLfP2IubT9S8v8pHcMwPP04sr5mqjvDXPId+iHFGR4kiSmIQK3cxoOINnGPpv", + "ML55pYHT9G3+7nJriJ5BeGf5JShM3pFEUByjkDPJY5ML9u08Sd4Ol9/yX56eQifzAsO82n87RO79fk5j", + "UrcqP6vQu4ixVOiFfSzS0QcueBybQudvNTxL+9uyDy6KJ6pXzPf4gpEbOyCdoLeldxhvGx5iOCR8rk/p", + "C1F+tzmTtdmL4kgA4EzdGgIVrH2PMGx1bc8TjJ2BNyVNy+cgZhmf+TVId7m4+DR/911BZZymbdHXLhOw", + "eJ4kK3AYdYpaH0iqiGfqH1JFRJhyZha7m5AbdXBo/qHwtSm+Vak+YjIU+0BlnzZ7QRWYEoMusbH51zxJ", + "AlMKJcG+RMUf/6ymPuCyPqZPpvR25vudcZdXMVVmX3oWU7s5bIZsyALhVd5emQbfvOTiUol/YTTcvCui", + "tAoK5SBYNF7A2RY52u/XmwA4yGJncN/ZfXlpxH1rpBGb2v2bp5ECP75xKgm5gHqQ0tVnuT/BWyWNo0Tu", + "HSgIURRa6Dqt9/L0dKuJaExJwUaSEd/VYRtH+c3fKVAj4/5RiykPhfMNrDIWaoJQjTq601kr9XPGPNOj", + "L6W1hNoHciEVSYzCPslieNgGUes2PwAu13boIqokZDnugsmqlNf/io3JRN+HKRF6bt0d0mYVuodPrT1X", + "OCffM0ODX4deC5kuQZXDqglqSyXWXZJLn+6U5+X84CU9BUW1WltCok5Mr02xGTSXKNY/tlZquqbwxKfO", + "fvDhlJWXVvG9ajU4myPzt8DhTmpszZVdunds7RkpE4vjP3DQPrbG01XXPE+/3/K2BtR3mfheysTg6Ml3", + "05kKHMKNK22VL7/8a8u7bL8zP07WuQsVDmeXLkX413GV2ozC66ZxG7wXRGn3FBHzpHfzNMnzpM/39NmG", + "BpzbAphOyo5P/y1gksl/a9j96WNcynC8U4TLRmnLPZf/amhr0zefXYML1y7D476QucE0txNIXlxWbUW5", + "OM1KhdYVHYFKSa5bXuWnWy7dZLLz5QpqUSwgrxLTv2J5WRyXHRAdnV10bRHoLpSZNiPYoip95K9eJBEW", + "xJUwumKKoxDHYRZjRVBexseU3pINbt1XpdJWn43eikk8B53XL5J5nZr7pGP4cQJOr1wIBzCuVEG5MbbU", + "FlPeSGSpvczuEFfqdvA9BK9FVGkJWG3S7pvmfXSepSkXSiJ1w6GOpQRfPiRZHPNoMUR5P4ZM6SPL4mzN", + "Gpt/nkRQN0T3Pa3k4i8N4HqmgvRSngLriMyTHQtjIx4tZ/lvSOSfy0efLzy2Ljp071oboLSW6nlU94jy", + "xPs2F7yGrYWXG6JVxndfoZO8+ECYScUTN+7JMergTPHelDAN3CLPfyr4nEb1cnFfSY2nU3xLkyzJC6Q+", + "ewzlJoUJ9YDCwxBo5HCK3IaERBIiP7buWA9quRSUPYsPy3n/6ZiY46aNMuUXjJkukhPqI9YypkNyxTmK", + "sZiSrW/mZaKlteJh4slx7VniPYz2njvsK+SMlvHd7VTalprm54jtzs0dm43svvx6tLBS/rZ7+LxwnouZ", + "TSHlXxcKDjZ3JWw6lPzyHlvttLY1r4HNDKBH9CHMcx7iGEVkTmKeQsVD0zboBpmIbf224fa2VtNircgN", + "DwYHg+D9m/f/PwAA///ivhr9rdAAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/init/main.go b/lib/system/init/main.go index d07e05f..43d2241 100644 --- a/lib/system/init/main.go +++ b/lib/system/init/main.go @@ -60,16 +60,20 @@ func main() { dropToShell() } - // Phase 7: Copy guest-agent to target location - if err := copyGuestAgent(log); err != nil { + // Phase 7: Copy guest-agent to target location (skips if already exists or skip_guest_agent=true) + if err := copyGuestAgent(log, cfg.SkipGuestAgent); err != nil { log.Error("agent", "failed to copy guest-agent", err) // Continue anyway - exec will still work, just no remote access } - // Phase 8: Setup kernel headers for DKMS - if err := setupKernelHeaders(log); err != nil { - log.Error("headers", "failed to setup kernel headers", err) - // Continue anyway - only needed for DKMS module building + // Phase 8: Setup kernel headers for DKMS (can be skipped via config) + if cfg.SkipKernelHeaders { + log.Info("headers", "skipping kernel headers setup (skip_kernel_headers=true)") + } else { + if err := setupKernelHeaders(log); err != nil { + log.Error("headers", "failed to setup kernel headers", err) + // Continue anyway - only needed for DKMS module building + } } // Phase 9: Mode-specific execution diff --git a/lib/system/init/mount.go b/lib/system/init/mount.go index fa1fc89..50ebc07 100644 --- a/lib/system/init/mount.go +++ b/lib/system/init/mount.go @@ -63,13 +63,36 @@ func mountEssentials(log *Logger) error { return nil } +// waitForDevice polls for a block device to become available. +// It polls every 10ms until the device exists or the timeout is reached. +func waitForDevice(device string, timeout time.Duration) error { + const pollInterval = 10 * time.Millisecond + deadline := time.Now().Add(timeout) + + for { + if _, err := os.Stat(device); err == nil { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for device %s", device) + } + time.Sleep(pollInterval) + } +} + // setupOverlay sets up the overlay filesystem: // - /dev/vda: readonly rootfs (ext4) // - /dev/vdb: writable overlay disk (ext4) // - /overlay/newroot: merged overlay filesystem func setupOverlay(log *Logger) error { - // Wait for block devices to be ready - time.Sleep(500 * time.Millisecond) + // Wait for block devices to be ready (polls every 10ms, 2s timeout) + // This typically completes in 10-50ms instead of always waiting 500ms + if err := waitForDevice("/dev/vda", 2*time.Second); err != nil { + return fmt.Errorf("wait for rootfs device: %w", err) + } + if err := waitForDevice("/dev/vdb", 2*time.Second); err != nil { + return fmt.Errorf("wait for overlay device: %w", err) + } // Create mount points for _, dir := range []string{"/lower", "/overlay"} { @@ -197,12 +220,27 @@ func redirectToConsole(device string) { } // copyGuestAgent copies the guest-agent binary to the target location in the new root. -func copyGuestAgent(log *Logger) error { +// It skips copying if: +// - skipGuestAgent config option is true +// - The destination file already exists (lazy copy optimization) +func copyGuestAgent(log *Logger, skipGuestAgent bool) error { const ( src = "/usr/local/bin/guest-agent" dst = "/overlay/newroot/opt/hypeman/guest-agent" ) + // Check for skip via config + if skipGuestAgent { + log.Info("agent", "skipping guest-agent copy (skip_guest_agent=true)") + return nil + } + + // Check if destination already exists (lazy copy - skip if already present) + if _, err := os.Stat(dst); err == nil { + log.Info("agent", "guest-agent already exists, skipping copy") + return nil + } + // Create target directory if err := os.MkdirAll("/overlay/newroot/opt/hypeman", 0755); err != nil { return fmt.Errorf("mkdir: %w", err) diff --git a/lib/vmconfig/config.go b/lib/vmconfig/config.go index bb7b9a0..df1bf6b 100644 --- a/lib/vmconfig/config.go +++ b/lib/vmconfig/config.go @@ -25,6 +25,10 @@ type Config struct { // Init mode: "exec" (default) or "systemd" InitMode string `json:"init_mode"` + + // Boot optimizations + SkipKernelHeaders bool `json:"skip_kernel_headers,omitempty"` + SkipGuestAgent bool `json:"skip_guest_agent,omitempty"` } // VolumeMount represents a volume mount configuration. diff --git a/openapi.yaml b/openapi.yaml index 1a6bc6f..4d4eef4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -179,6 +179,23 @@ components: enum: [cloud-hypervisor, qemu] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor + skip_kernel_headers: + type: boolean + description: | + Skip kernel headers installation during boot for faster startup. + When true, DKMS (Dynamic Kernel Module Support) will not work, + preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). + Recommended for workloads that don't need kernel module compilation. + default: false + example: true + skip_guest_agent: + type: boolean + description: | + Skip guest-agent installation during boot. + When true, the exec and stat APIs will not work for this instance. + The instance will still run, but remote command execution will be unavailable. + default: false + example: false # Future: port_mappings, timeout_seconds Instance: From f5fd27a64afa0fdafefe574ab2bdc3a39ecd16e6 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Fri, 23 Jan 2026 14:00:58 -0500 Subject: [PATCH 2/4] fix: skip systemd agent service injection when skip_guest_agent is true When skip_guest_agent is true, the guest-agent binary is not copied, but the systemd service was still being injected. This caused systemd to repeatedly try to start the non-existent binary every 3 seconds, resulting in continuous log spam. Now the service injection is skipped when skip_guest_agent is true. --- lib/system/init/mode_systemd.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/system/init/mode_systemd.go b/lib/system/init/mode_systemd.go index cc6e59f..744d7ed 100644 --- a/lib/system/init/mode_systemd.go +++ b/lib/system/init/mode_systemd.go @@ -17,11 +17,15 @@ import ( func runSystemdMode(log *Logger, cfg *vmconfig.Config) { const newroot = "/overlay/newroot" - // Inject hypeman-agent.service - log.Info("systemd", "injecting hypeman-agent.service") - if err := injectAgentService(newroot); err != nil { - log.Error("systemd", "failed to inject service", err) - // Continue anyway - VM will work, just without agent + // Inject hypeman-agent.service (skip if guest-agent was not copied) + if cfg.SkipGuestAgent { + log.Info("systemd", "skipping agent service injection (skip_guest_agent=true)") + } else { + log.Info("systemd", "injecting hypeman-agent.service") + if err := injectAgentService(newroot); err != nil { + log.Error("systemd", "failed to inject service", err) + // Continue anyway - VM will work, just without agent + } } // Change root to the new filesystem using chroot From 73a725221b03d6ee18a49e88000b1aba0e0a260d Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Fri, 23 Jan 2026 14:40:42 -0500 Subject: [PATCH 3/4] fix: skip guest-agent start in exec mode when skip_guest_agent is true When skip_guest_agent is true, the guest-agent binary is not copied, but exec mode was still unconditionally trying to start it. This caused an error log on every boot with skip_guest_agent enabled. Now the guest-agent start is skipped when skip_guest_agent is true, consistent with the systemd mode behavior. --- lib/system/init/mode_exec.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/system/init/mode_exec.go b/lib/system/init/mode_exec.go index 5a9e19a..549bb59 100644 --- a/lib/system/init/mode_exec.go +++ b/lib/system/init/mode_exec.go @@ -36,13 +36,18 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") os.Setenv("HOME", "/root") - // Start guest-agent in background - log.Info("exec", "starting guest-agent in background") - agentCmd := exec.Command("/opt/hypeman/guest-agent") - agentCmd.Stdout = os.Stdout - agentCmd.Stderr = os.Stderr - if err := agentCmd.Start(); err != nil { - log.Error("exec", "failed to start guest-agent", err) + // Start guest-agent in background (skip if guest-agent was not copied) + var agentCmd *exec.Cmd + if cfg.SkipGuestAgent { + log.Info("exec", "skipping guest-agent (skip_guest_agent=true)") + } else { + log.Info("exec", "starting guest-agent in background") + agentCmd = exec.Command("/opt/hypeman/guest-agent") + agentCmd.Stdout = os.Stdout + agentCmd.Stderr = os.Stderr + if err := agentCmd.Start(); err != nil { + log.Error("exec", "failed to start guest-agent", err) + } } // Build the entrypoint command @@ -94,7 +99,7 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { // Wait for guest-agent (keeps init alive, prevents kernel panic) // The guest-agent runs forever, so this effectively keeps the VM alive // until it's explicitly terminated - if agentCmd.Process != nil { + if agentCmd != nil && agentCmd.Process != nil { agentCmd.Wait() } From 2d95b496df661cfca3d89fcf424172eb247eb1b7 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Fri, 23 Jan 2026 14:57:21 -0500 Subject: [PATCH 4/4] fix: add device polling for config disk to prevent race condition The device polling optimization waited only for /dev/vda and /dev/vdb, but /dev/vdc (config disk) was mounted without polling. This could cause mount failures if vdc appeared slightly after vdb. Now readConfig() polls for /dev/vdc before attempting to mount it, consistent with how we handle the other block devices. --- lib/system/init/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/system/init/config.go b/lib/system/init/config.go index bf6533b..35c9786 100644 --- a/lib/system/init/config.go +++ b/lib/system/init/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "time" "github.com/kernel/hypeman/lib/vmconfig" ) @@ -19,6 +20,11 @@ func readConfig(log *Logger) (*vmconfig.Config, error) { return nil, fmt.Errorf("mkdir config mount: %w", err) } + // Wait for config disk to be ready (polls every 10ms, 2s timeout) + if err := waitForDevice("/dev/vdc", 2*time.Second); err != nil { + return nil, fmt.Errorf("wait for config device: %w", err) + } + // Mount config disk (/dev/vdc) read-only cmd := exec.Command("/bin/mount", "-o", "ro", "/dev/vdc", configMount) if output, err := cmd.CombinedOutput(); err != nil {