diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 3cd61bb9..efb65eb1 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -10,20 +10,22 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" ) // ApiService implements the oapi.StrictServerInterface type ApiService struct { - Config *config.Config - ImageManager images.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager + Config *config.Config + ImageManager images.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager } var _ oapi.StrictServerInterface = (*ApiService)(nil) @@ -39,16 +41,18 @@ func New( ingressManager ingress.Manager, buildManager builds.Manager, resourceManager *resources.Manager, + vmMetricsManager *vm_metrics.Manager, ) *ApiService { return &ApiService{ - Config: config, - ImageManager: imageManager, - InstanceManager: instanceManager, - VolumeManager: volumeManager, - NetworkManager: networkManager, - DeviceManager: deviceManager, - IngressManager: ingressManager, - BuildManager: buildManager, - ResourceManager: resourceManager, + Config: config, + ImageManager: imageManager, + InstanceManager: instanceManager, + VolumeManager: volumeManager, + NetworkManager: networkManager, + DeviceManager: deviceManager, + IngressManager: ingressManager, + BuildManager: buildManager, + ResourceManager: resourceManager, + VMMetricsManager: vmMetricsManager, } } diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e0cd9c74..61280f39 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -274,6 +275,52 @@ func (s *ApiService) GetInstance(ctx context.Context, request oapi.GetInstanceRe return oapi.GetInstance200JSONResponse(instanceToOAPI(*inst)), nil } +// GetInstanceStats returns resource utilization statistics for an instance +// The id parameter can be an instance ID, name, or ID prefix +// Note: Resolution is handled by ResolveResource middleware +func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInstanceStatsRequestObject) (oapi.GetInstanceStatsResponseObject, error) { + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.GetInstanceStats500JSONResponse{ + Code: "internal_error", + Message: "resource not resolved", + }, nil + } + + // Build instance info for metrics collection + info := vm_metrics.BuildInstanceInfo( + inst.Id, + inst.Name, + inst.HypervisorPID, + inst.NetworkEnabled, + inst.Vcpus, + inst.Size+inst.HotplugSize, + ) + + // Collect stats using vm_metrics manager + vmStats := s.VMMetricsManager.GetInstanceStats(ctx, info) + + // Map domain type to API type + return oapi.GetInstanceStats200JSONResponse(vmStatsToOAPI(vmStats)), nil +} + +// vmStatsToOAPI converts vm_metrics.VMStats to oapi.InstanceStats +func vmStatsToOAPI(s *vm_metrics.VMStats) oapi.InstanceStats { + stats := oapi.InstanceStats{ + InstanceId: s.InstanceID, + InstanceName: s.InstanceName, + CpuSeconds: s.CPUSeconds(), + MemoryRssBytes: int64(s.MemoryRSSBytes), + MemoryVmsBytes: int64(s.MemoryVMSBytes), + NetworkRxBytes: int64(s.NetRxBytes), + NetworkTxBytes: int64(s.NetTxBytes), + AllocatedVcpus: s.AllocatedVcpus, + AllocatedMemoryBytes: s.AllocatedMemoryBytes, + MemoryUtilizationRatio: s.MemoryUtilizationRatio(), + } + return stats +} + // DeleteInstance stops and deletes an instance // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 6737b051..93b307de 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -19,25 +19,27 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" ) // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } // initializeApp is the injector function @@ -56,6 +58,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideIngressManager, providers.ProvideBuildManager, providers.ProvideResourceManager, + providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, wire.Struct(new(application), "*"), diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 5ac1bda9..c0a28717 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -20,6 +20,7 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" "log/slog" ) @@ -63,26 +64,31 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager) + if err != nil { + return nil, nil, err + } registry, err := providers.ProvideRegistry(paths, manager) if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager) + apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, vm_metricsManager) mainApplication := &application{ - Ctx: context, - Logger: logger, - Config: config, - ImageManager: manager, - SystemManager: systemManager, - NetworkManager: networkManager, - DeviceManager: devicesManager, - InstanceManager: instancesManager, - VolumeManager: volumesManager, - IngressManager: ingressManager, - BuildManager: buildsManager, - ResourceManager: resourcesManager, - Registry: registry, - ApiService: apiService, + Ctx: context, + Logger: logger, + Config: config, + ImageManager: manager, + SystemManager: systemManager, + NetworkManager: networkManager, + DeviceManager: devicesManager, + InstanceManager: instancesManager, + VolumeManager: volumesManager, + IngressManager: ingressManager, + BuildManager: buildsManager, + ResourceManager: resourcesManager, + VMMetricsManager: vm_metricsManager, + Registry: registry, + ApiService: apiService, } return mainApplication, func() { }, nil @@ -92,18 +98,19 @@ func initializeApp() (*application, func(), error) { // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } diff --git a/dashboards/hypeman.json b/dashboards/hypeman.json index c2c46c55..6e09676e 100644 --- a/dashboards/hypeman.json +++ b/dashboards/hypeman.json @@ -417,6 +417,112 @@ ], "title": "Exec Sessions & Duration", "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, + "id": 18, + "title": "VM Resource Utilization", + "type": "row" + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "id": 19, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_cpu_seconds_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}}", + "refId": "A" + } + ], + "title": "VM CPU Usage (cores)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "short", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "id": 20, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_rss_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}} RSS", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_vms_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}} VMS", + "refId": "B" + } + ], + "title": "VM Memory Usage (RSS & VMS)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "bytes", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "id": 21, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_network_rx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}} RX", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_network_tx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}} TX", + "refId": "B" + } + ], + "title": "VM Network I/O", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "Bps", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }, + "id": 23, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_utilization_ratio{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}}", + "refId": "A" + } + ], + "title": "VM Memory Utilization (% of allocated)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, + "overrides": [] + } } ], "refresh": "10s", diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index d47bfa34..7357ac8f 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -122,6 +122,10 @@ func (m *mockInstanceManager) ListInstanceAllocations(ctx context.Context) ([]re return nil, nil } +func (m *mockInstanceManager) ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) { + return nil, nil +} + // mockVolumeManager implements volumes.Manager for testing type mockVolumeManager struct { volumes map[string]*volumes.Volume diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 84daaa1e..09c9616e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -38,6 +38,9 @@ type Manager interface { // ListInstanceAllocations returns resource allocations for all instances. // Used by the resource manager for capacity tracking. ListInstanceAllocations(ctx context.Context) ([]resources.InstanceAllocation, error) + // ListRunningInstancesInfo returns info needed for utilization metrics collection. + // Used by the resource manager for VM utilization tracking. + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) } // ResourceLimits contains configurable resource limits for instances @@ -328,3 +331,39 @@ func (m *manager) ListInstanceAllocations(ctx context.Context) ([]resources.Inst return allocations, nil } + +// ListRunningInstancesInfo returns info needed for utilization metrics collection. +// Used by the resource manager for VM utilization tracking. +func (m *manager) ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return nil, err + } + + infos := make([]resources.InstanceUtilizationInfo, 0, len(instances)) + for _, inst := range instances { + // Only include running instances (they have a hypervisor process) + if inst.State != StateRunning { + continue + } + + info := resources.InstanceUtilizationInfo{ + ID: inst.Id, + Name: inst.Name, + HypervisorPID: inst.HypervisorPID, + // Include allocated resources for utilization ratio calculations + AllocatedVcpus: inst.Vcpus, + AllocatedMemoryBytes: inst.Size + inst.HotplugSize, + } + + // Derive TAP device name if networking is enabled + if inst.NetworkEnabled { + info.TAPDevice = network.GenerateTAPName(inst.Id) + } + + infos = append(infos, info) + } + + return infos, nil +} + diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 67037b33..3771bd17 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -1001,3 +1001,4 @@ func (r *testInstanceResolver) ResolveInstance(ctx context.Context, nameOrID str // For tests, just return nameOrID as both name and id return nameOrID, nameOrID, nil } + diff --git a/lib/network/allocate.go b/lib/network/allocate.go index b622ace4..aaf36f82 100644 --- a/lib/network/allocate.go +++ b/lib/network/allocate.go @@ -53,7 +53,7 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N } // 5. Generate TAP name (tap-{first8chars-of-id}) - tap := generateTAPName(req.InstanceID) + tap := GenerateTAPName(req.InstanceID) // 6. Create TAP device with bidirectional rate limiting if err := m.createTAPDevice(tap, network.Bridge, network.Isolated, req.DownloadBps, req.UploadBps, req.UploadCeilBps); err != nil { @@ -269,8 +269,9 @@ func generateMAC() (string, error) { // TAPPrefix is the prefix used for hypeman TAP devices const TAPPrefix = "hype-" -// generateTAPName generates TAP device name from instance ID -func generateTAPName(instanceID string) string { +// GenerateTAPName generates TAP device name from instance ID. +// Exported for use by other packages (e.g., vm_metrics). +func GenerateTAPName(instanceID string) string { // Use first 8 chars of instance ID // hype-{8chars} fits within 15-char Linux interface name limit shortID := instanceID diff --git a/lib/network/bridge.go b/lib/network/bridge.go index ca150d8d..a979c111 100644 --- a/lib/network/bridge.go +++ b/lib/network/bridge.go @@ -803,7 +803,7 @@ func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs [] // Build set of expected TAP names for running instances expectedTAPs := make(map[string]bool) for _, id := range runningInstanceIDs { - tapName := generateTAPName(id) + tapName := GenerateTAPName(id) expectedTAPs[tapName] = true } diff --git a/lib/network/derive.go b/lib/network/derive.go index 591bab3e..01a81f9b 100644 --- a/lib/network/derive.go +++ b/lib/network/derive.go @@ -52,7 +52,7 @@ func (m *manager) deriveAllocation(ctx context.Context, instanceID string) (*All // 4. Use stored metadata to derive allocation (works for all hypervisors) if meta.IP != "" && meta.MAC != "" { - tap := generateTAPName(instanceID) + tap := GenerateTAPName(instanceID) // Determine state based on socket existence and snapshot socketPath := m.paths.InstanceSocket(instanceID, hypervisor.SocketNameForType(hypervisor.Type(meta.HypervisorType))) diff --git a/lib/network/manager_test.go b/lib/network/manager_test.go index b6c74ef3..9081528c 100644 --- a/lib/network/manager_test.go +++ b/lib/network/manager_test.go @@ -53,7 +53,7 @@ func TestGenerateTAPName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := generateTAPName(tt.instanceID) + got := GenerateTAPName(tt.instanceID) assert.Equal(t, tt.want, got) // Verify within Linux interface name limit (15 chars) assert.LessOrEqual(t, len(got), 15) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 90d86f75..5b439652 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -621,6 +621,39 @@ type InstanceGPU struct { // - Unknown: Failed to determine state (see state_error for details) type InstanceState string +// InstanceStats Real-time resource utilization statistics for a VM instance +type InstanceStats struct { + // AllocatedMemoryBytes Total memory allocated to the VM (Size + HotplugSize) in bytes + AllocatedMemoryBytes int64 `json:"allocated_memory_bytes"` + + // AllocatedVcpus Number of vCPUs allocated to the VM + AllocatedVcpus int `json:"allocated_vcpus"` + + // CpuSeconds Total CPU time consumed by the VM hypervisor process in seconds + CpuSeconds float64 `json:"cpu_seconds"` + + // InstanceId Instance identifier + InstanceId string `json:"instance_id"` + + // InstanceName Instance name + InstanceName string `json:"instance_name"` + + // MemoryRssBytes Resident Set Size - actual physical memory used by the VM in bytes + MemoryRssBytes int64 `json:"memory_rss_bytes"` + + // MemoryUtilizationRatio Memory utilization ratio (RSS / allocated memory). Only present when allocated_memory_bytes > 0. + MemoryUtilizationRatio *float64 `json:"memory_utilization_ratio"` + + // MemoryVmsBytes Virtual Memory Size - total virtual memory allocated in bytes + MemoryVmsBytes int64 `json:"memory_vms_bytes"` + + // NetworkRxBytes Total network bytes received by the VM (from TAP interface) + NetworkRxBytes int64 `json:"network_rx_bytes"` + + // NetworkTxBytes Total network bytes transmitted by the VM (from TAP interface) + NetworkTxBytes int64 `json:"network_tx_bytes"` +} + // PassthroughDevice Physical GPU available for passthrough type PassthroughDevice struct { // Available Whether this GPU is available (not attached to an instance) @@ -1026,6 +1059,9 @@ type ClientInterface interface { // StatInstancePath request StatInstancePath(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetInstanceStats request + GetInstanceStats(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StopInstance request StopInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1439,6 +1475,18 @@ func (c *Client) StatInstancePath(ctx context.Context, id string, params *StatIn return c.Client.Do(req) } +func (c *Client) GetInstanceStats(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetInstanceStatsRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) StopInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewStopInstanceRequest(c.Server, id) if err != nil { @@ -2591,6 +2639,40 @@ func NewStatInstancePathRequest(server string, id string, params *StatInstancePa return req, nil } +// NewGetInstanceStatsRequest generates requests for GetInstanceStats +func NewGetInstanceStatsRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/stats", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewStopInstanceRequest generates requests for StopInstance func NewStopInstanceRequest(server string, id string) (*http.Request, error) { var err error @@ -3017,6 +3099,9 @@ type ClientWithResponsesInterface interface { // StatInstancePathWithResponse request StatInstancePathWithResponse(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*StatInstancePathResponse, error) + // GetInstanceStatsWithResponse request + GetInstanceStatsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceStatsResponse, error) + // StopInstanceWithResponse request StopInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StopInstanceResponse, error) @@ -3726,6 +3811,30 @@ func (r StatInstancePathResponse) StatusCode() int { return 0 } +type GetInstanceStatsResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InstanceStats + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetInstanceStatsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetInstanceStatsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type StopInstanceResponse struct { Body []byte HTTPResponse *http.Response @@ -4205,6 +4314,15 @@ func (c *ClientWithResponses) StatInstancePathWithResponse(ctx context.Context, return ParseStatInstancePathResponse(rsp) } +// GetInstanceStatsWithResponse request returning *GetInstanceStatsResponse +func (c *ClientWithResponses) GetInstanceStatsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceStatsResponse, error) { + rsp, err := c.GetInstanceStats(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetInstanceStatsResponse(rsp) +} + // StopInstanceWithResponse request returning *StopInstanceResponse func (c *ClientWithResponses) StopInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StopInstanceResponse, error) { rsp, err := c.StopInstance(ctx, id, reqEditors...) @@ -5469,6 +5587,46 @@ func ParseStatInstancePathResponse(rsp *http.Response) (*StatInstancePathRespons return response, nil } +// ParseGetInstanceStatsResponse parses an HTTP response from a GetInstanceStatsWithResponse call +func ParseGetInstanceStatsResponse(rsp *http.Response) (*GetInstanceStatsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetInstanceStatsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InstanceStats + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseStopInstanceResponse parses an HTTP response from a StopInstanceWithResponse call func ParseStopInstanceResponse(rsp *http.Response) (*StopInstanceResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -5896,6 +6054,9 @@ type ServerInterface interface { // Get filesystem path info // (GET /instances/{id}/stat) StatInstancePath(w http.ResponseWriter, r *http.Request, id string, params StatInstancePathParams) + // Get instance resource utilization stats + // (GET /instances/{id}/stats) + GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) StopInstance(w http.ResponseWriter, r *http.Request, id string) @@ -6094,6 +6255,12 @@ func (_ Unimplemented) StatInstancePath(w http.ResponseWriter, r *http.Request, w.WriteHeader(http.StatusNotImplemented) } +// Get instance resource utilization stats +// (GET /instances/{id}/stats) +func (_ Unimplemented) GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) func (_ Unimplemented) StopInstance(w http.ResponseWriter, r *http.Request, id string) { @@ -6945,6 +7112,37 @@ func (siw *ServerInterfaceWrapper) StatInstancePath(w http.ResponseWriter, r *ht handler.ServeHTTP(w, r) } +// GetInstanceStats operation middleware +func (siw *ServerInterfaceWrapper) GetInstanceStats(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetInstanceStats(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // StopInstance operation middleware func (siw *ServerInterfaceWrapper) StopInstance(w http.ResponseWriter, r *http.Request) { @@ -7375,6 +7573,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}/stat", wrapper.StatInstancePath) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/instances/{id}/stats", wrapper.GetInstanceStats) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/stop", wrapper.StopInstance) }) @@ -8529,6 +8730,41 @@ func (response StatInstancePath500JSONResponse) VisitStatInstancePathResponse(w return json.NewEncoder(w).Encode(response) } +type GetInstanceStatsRequestObject struct { + Id string `json:"id"` +} + +type GetInstanceStatsResponseObject interface { + VisitGetInstanceStatsResponse(w http.ResponseWriter) error +} + +type GetInstanceStats200JSONResponse InstanceStats + +func (response GetInstanceStats200JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceStats404JSONResponse Error + +func (response GetInstanceStats404JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceStats500JSONResponse Error + +func (response GetInstanceStats500JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type StopInstanceRequestObject struct { Id string `json:"id"` } @@ -8932,6 +9168,9 @@ type StrictServerInterface interface { // Get filesystem path info // (GET /instances/{id}/stat) StatInstancePath(ctx context.Context, request StatInstancePathRequestObject) (StatInstancePathResponseObject, error) + // Get instance resource utilization stats + // (GET /instances/{id}/stats) + GetInstanceStats(ctx context.Context, request GetInstanceStatsRequestObject) (GetInstanceStatsResponseObject, error) // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) StopInstance(ctx context.Context, request StopInstanceRequestObject) (StopInstanceResponseObject, error) @@ -9729,6 +9968,32 @@ func (sh *strictHandler) StatInstancePath(w http.ResponseWriter, r *http.Request } } +// GetInstanceStats operation middleware +func (sh *strictHandler) GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) { + var request GetInstanceStatsRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetInstanceStats(ctx, request.(GetInstanceStatsRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetInstanceStats") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetInstanceStatsResponseObject); ok { + if err := validResponse.VisitGetInstanceStatsResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // StopInstance operation middleware func (sh *strictHandler) StopInstance(w http.ResponseWriter, r *http.Request, id string) { var request StopInstanceRequestObject @@ -9961,151 +10226,159 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XITubboq6j6nl3HOdt2nA+Y4FNTt0ICTPYhkEtI9j17wjVyt2xr0i31SGoHQ/F3", - "HmAecZ7klpak/rLa7gAx5MCuXTUmrc+lpfWttT4EIU9SzghTMhh+CGQ4IwmGn4dK4XB2yeMsIa/I7xmR", - "Sv85FTwlQlECjRKeMTVKsZrpf0VEhoKminIWDIMzrGboZkYEQXMYBckZz+IIjQmCfiQKugF5h5M0JsEw", - "2E6Y2o6wwkE3UItU/0kqQdk0+NgNBMERZ/HCTDPBWayC4QTHknRr057qoRGWSHfpQZ98vDHnMcEs+Agj", - "/p5RQaJg+Gt5G2/yxnz8GwmVnvxwjmmMxzE5JnMakmUwhJkQhKlRJOiciGVQHJnv8QKNecYiZNqhDsvi", - "GNEJYpyRrQow2JxGVENCN9FTB0MlMuKBTARrGtHIcwJHJ8h8RifHqDMj76qT7P40Pgiah2Q4IcuD/pIl", - "mPU0cPWy3PjQtjz2833fyJQnSTaaCp6lyyOfvDw9vUDwEbEsGRNRHvFgNx+PMkWmROgB05COcBQJIqV/", - "/+5jeW2DwWAwxLvDwaA/8K1yTljERSNIzWc/SHcGEVkxZCuQ2vGXQPri8uT45BAdcZFygaHv0kw1xC6D", - "p7yvMtpUT8WH/48zGkcerOd6YYpEI6yWNwWdkG1DOUOKJkQqnKRBN5hwkehOQYQV6ekvbVA9FASvmU63", - "aDXZMtJnBqajRDaN7pogylBC45hKEnIWyfIclKmH+82bKaEuEYJ7aMUT/WeUECnxlKCOJmCaijIkFVaZ", - "RFSiCaYxibbagMyHw2Yzv/ExohFhik5o9aYFY92gh8fhzu6e9xYneEpGEZ1anlAd/hj+jvgE6XEUgtb+", - "jWiUX7TbB0wpyGR5vqdARGESQSZEEBZ+9nSp4HPCMDPE/t9g3uB/bRfMcttyym0A5lnR/GM3+D0jGRml", - "XFKzwiUaYr9oNAJQI+jhXzN8WnXWJYySCovV9wNafIGbaNbXCjbnpmmdMgHhscNUbnYjAXoyJ0z5qBBT", - "9kN1x8/5FMWUEWRbWPhOuEB6gp9jPt0KvszeukEB0uULrdf9CQTJ/KFhNP2tGxCWJRqYMZ+WoTkjWKgx", - "qQCzgUHYgYrVNYL/rHIlqmcwxpKMVlOFM8oYiZBuaS+raYkyCXLg0vbhZlxTNZoTIb33CJb1X1Qh26Jx", - "qJiH1xMak9EMy5lZMY4iuIM4PqvsxCMLVYRLnGrC5gYEHi2R4uj8l8PdBw+RncADQ8kzEZoVLO+k1FsP", - "b9oihcUYx7EXN5rR7fZ8dxlD/Bhwnl+MJn6SY6BDTEO9AnuaevhukGZyZn4BPdarAn6myYBGr1j/fuPZ", - "9BEQCSODN2okfgnrZWoOG01jrmG6QBmjv2cV8bWPTrQkrpAm/jQiURdh+KDJMM4U700JI0LTKTQRPEFq", - "RlBJxEQd0p/2u+hKS109LWP28G5vMOgNroKqkBjv96ZppkGBlSJCL/D//Yp77w97/xr0Hr0pfo76vTd/", - "/zcfArSVezU66XXafXbc3e8it9iyMFxf6GpBeYWs6aMi5vhO9N2/7ekdnSwzeLP+iIfXRPQp347pWGCx", - "2GZTyt4NY6yIVNXdrG67dn+wthUbY1O99VturSb6A7p1Yn5DRKgpZUw0gsiuJpZUyS7CWnsEIoM0N/tP", - "FGKmcdYwdi4QYRG6oWqGMLSrQiBZ9HBKe9QsNegGCX73nLCpVt8f7i3ho0bGjv3Re/Mf7k9b/9uLkiKL", - "iQcZX/FMUTZF8Nlw3xmVqFgDVSRZy24ddLMYRKyEshPTbSdfCRYCL/yn5ha36vSk0sSn8fjMBfLs79gp", - "2BJZpQ0YAgbzCez32dnFtr6SKZZSzQTPprPyqfzq6MGbEiwapAG3yW4QUXk9onw0Tn1rovIanWy/RJpa", - "oZgmVBXUaWcwOH28La8C/Y8H7h9bfXRs7CqwfL15LizRlDMsCLDuCHGGjs4uEI5jHlplaKIlrAmdZoJE", - "/Zo2DKP7sIWw+Wfw4SdsTgVniZaF5lhQfXkqOv6H4MXL4yejJy8ug6E+ySgLrcJ89vLV62AY7A0Gg8DH", - "6vRJrEHGZ2cXR7Bj3X7GVRpn05Gk70nFOhXsPXsc1Bd+mO8XJSThwsijdgzUmVXJgWHXKKbXBF3p8cyh", - "7TyrE+pdmGoJaLNFSsScSp+e+Uv+TZ93Jkn5bprLUEUJScSciPys4fD7JV4fxjyLeqUpu8HvJAG0Lhbq", - "aeTX9VpxgTXkHccpZaSRvne/FZp8w8V1zHHU2/nCJJkRpcde3uIL86F6mBYBSH7+QXdJzmfRDY3UbBTx", - "G6aX7KE99gvKG+cE6J3eCY7/+uPPy9NCANl5Nk4tNdrZffCZ1KhGf/TQXuUi30iW+rdxkfo3cXn61x9/", - "up183U0QpvEzqhAdo69Xt/LPGVEzIkpcyR2w/pORDqE7cvhSmr5iACjbz5cIJ58TEeOFhxDuDDyU8J+C", - "Krhfth/SHA3pzmvIoB7NMa9lQjjwU0LPojxreqzvt6XLbVaSL2Rn99T+3G1Lm+dh6pQpu6Td+nJegBFc", - "i/BzKlSGY40nFTbntYkbb4tHLDDOnLJ4Ys8/xwesqibUtuKZGRlcL8vCil8iM1S+WSJb43nyGTZzLS/M", - "pOJJybyJOjUFjlZVveqJzXnci7DCQI9bMg2z3GWjfbIwQ5lDaULN0XTssQpoDKQMTekUjxeqKuDsDJaP", - "3g9oN74P1E0OLYMeJBop7vHTOGw5OdZwdG3b2A3B/TVSfDSfUM/IOaUqNFYqUVjznlmk1UP00pBab1oX", - "3cyopm0SOSAAQ7s8LQve/SvWQ3pxQ3ScT5APmw+pWTpYJ2CIDhelRVAwNKHxYgthdHnaR6/z1f67RAwr", - "OifOwzfDEo0JYSgDnkgimB/8luUFZFJrSFTVu1uZ3TgDt0C/4PZbH2kBLsEM3dA4BvtEghUNwbgxprX9", - "gFHZHJSeSRMAVoh5V6yMWdarWif5q90vr8iUSiVqzhfUefX0aG9v71GdSO8+6A12ejsPXu8MhgP9/3+1", - "99N8eX+nb6zDKr2w5qIyRTm6ODnetRyhOo96v48fHbx7h9Wjh/RGPnqfjMX0tz28EY+onzwdF3Yu1Mkk", - "ET1H+jRW+axbJSNSg/Xqk41St3LGOjP4KvZjdvdat7wL963PdWEN57d3sNaJ4FrnR2lzS/vRf9XyQYH5", - "JYXM2hhD6rWmHlN5/VgQfK1FeQ9/1exZjgzf8RsYMi28jheIvNNyLYmQ4FxNpFHSqmLKzv5P+wd7D/cP", - "BgOPr3QZiXlIR6HmKq0WoDXDGC+0cqr7oA5I1xEax3xcRd4Hew8Pfho82tltuw4jm7aDQy5FuV6oYyHy", - "dxcB475UFrW7+9PDvb29wcOHu/utVmUFvFaLcsJgRXT4ae+n/Z2D3f1WUPDJ+k+c77rui4s8SHqYpjE1", - "mk1PpiSkExoi8H4j3QF1EmBLJBezq3dyjKORsGKglx8oTGMPGEqmFjOZbYk6mqcnWaxoGhPzDQ6klaQL", - "Oz+GkXxmOcoYEaPctX+LkazHf605wu0lbwIiSkTG2XRq3CoF6E6pBMmiEIgoiaOhuaFr6RycZrGwN014", - "YPfQEhue8xsiejGZk7iMBIYd6cUmXBCU44k5tMquKJvjmEYjytLMixKNoHyaCZAvzaAIj3mmQJY0B1ae", - "BPwUoCNMNLlu5yYrDINLUz87u7ittSUVfEJjzzbmejD71bJ0Z4d4vj847+38HzA+vGTxwtAByhD0SXhE", - "+rUwLWjfentnTWvKY+RQeXVLe8KumccmlWu7DiISqZnWSDFDY4IsmzSWNLBTFpMUBP6Rj2BOBE7IOJtM", - "iBglHk3rqf6OTAOj+FOGTh9XiaYmzm3FrbPK4YC8NcEhZdOt1tD3aHK1bXRL0HzjP65XxPiSm1y3+qiE", - "bWO9t330Io9KRM/OLiTKZ+l7VLyWXpKz2UJq5cSMaCIxKCtrZoCcrcnwWdHR6rAeYpx4CZC7CKgzn6YZ", - "XMPzV72Tl5fbSUTm3cqa9MebGY+JXvdWSbaaOwdu4dKp2LnnTSKyQQzZ9gKVYJXf4NZAKt1XD3QUVzge", - "yZgrz2pe648IPqLO5VPjuNMr6KK0cpT67yUoVPD7offGaIrUNO05TFjXtSsXfK3ZIzFsq7y9yqS+q/IL", - "wbGJYa7icxEL5A6eX1cPml+vvb12EN+8J87VUeOciUd3OTo9NppZyJnClBGBEqKwjZguuRPBqx10g54W", - "BiJMEs4Qn0z+c7WDscF2k6PLKu3/aCns8k40/4bQIk3k4jmJUIIZnRCpbGhRZWY5w7sPHg5NUGNEJvsP", - "Hvb7fb9ZXYlFyqkvpuxJ/q3dUWwbp1SvGLMvZ593DnfgOG2zlw/B2eHrX4JhsJ1JsR3zEMfbckzZsPTv", - "/J/FB/hh/jmmzOtwbRUHSydL8a+V4001zzJ/H+qdMBLmCMlBSlxrm/Rz8hcaNWP6nkTIG4ai8BRp+Rsw", - "7vPiTT4jcrR4SKBKEaNlN0GL6FH6frW67QQjaGPnzJiicRFYu6xof1JotFwZabYUZZYSlseWxbH5FXI2", - "17fCF2hWIeDu29Jh3HBxTdl0FFEPdv7TfEQRFSRU4Mdff4eCbZym61HRL/zlNK1t0KwNmfFwl69OyT/F", - "4Fqd/eX0H7//X3n20287vz+/vPzv+bN/HL+g/30Zn738LD//6miprxrytNKnBlbGSqhTW/Q4xSr0CD4z", - "LlUD1OwXpDhKdOc+OgIFbXjFeug5VUTgeIiuApzSvgVmP+TJVYA65B0OlemFOEN6KDQjOCJiS3c+M7EO", - "uvMHpwN+rI8RLRhOaIiEBXLuQ5fZOOIJpmzril0xOxZyG5HgtNG/IhTiVGWC6BPRsma8QGOBtbpp1ehi", - "8i76gNP049YVA02UvFNC7yDFQuWhlW4GOGi7KuMUss1JhOY4zoi0muwVy/kHqOZ6EIXFlKh+bkIEQ03N", - "MdMAFK+awYWq+JYPBl3POSLdTh9kTKUiDOVWCSoBeVHHRQYcDCrX/2BwsN7/mOPQCvQD7F5+VuiQssX9", - "MAgMUxtiPJopla5/Jwj0xtwR9Mvr12caDPq/58gNVMAiP2KjjOE0jSmRxqumYpBJbDDGVuDznJnTbbmh", - "16ax7hbL9ft4AhOj18/PkSIioczQ706owTmhod4f+HeolJlGRYrR4dHpk61+i3eRANt8/SvO8XW+w5ob", - "wRm3ljVM6FEYzTV8u+jkuKvFKXtDC0EL/KZPuUCxITDFvR6iC0mqUQxwVMbFY04yXhQWMkPVr4ItN2Ja", - "pxRD9CqX73C+lDzku0AGN2RxL2HYK/ZPjRjGqbs0ere6VnBXW/3FkjZw4WKFrNEbWHEzKVh9/T0QhzvP", - "Wd32eLu7XTZa6sn8qFGc/Z1LIHu31SVvGz5bjQQqRX7lEbRfN/T1UwJZ3Qk9O7uAcFEsR5LhVM64ag7O", - "wMi1QeQdlUouB462CidYDpytsicTErsiGutLhsCKjDGIjKhv44sHt37NWINvL7B2ZSjs58azWgHtjsJZ", - "GwmCLxS0ShvMn79sYOqdLKcSYuojBmU+5gLBPjmqtBtQTxDMoZR0ykiETs6Kp1aFwcMNX9vTo93+zsOD", - "/s5g0N8ZtDH/JDhcMffp4VH7yQe7RiEe4vEwjIZk8hnmJ4vYRuDA8Q1eSHTlRMKrwMigJeGzdG2t2NjK", - "tbccvPtpsbp1JrguGvc20bet6P2qN9Dn1dfPreWKB//6rIfSpC0bPofGrtfoNoZRgkKexRH7d4XG+uYZ", - "VYBEVmORRBUPy+GyXrBrxm9YdevGPqbv7+8ZEQt0eXpasaYKMrFvbFtsnKdp4znw9FbHsLtGvFu7mlKw", - "9SYCrOuUsMSBvng4ddn04+I6DNa1MAEV4p/XTUqZAbc++xV7qinvEZmPsswn6OhPLkLz4uLkuHLgGD/c", - "ORgcPOodjHce9vajwU4P7+w97O0+wIPJXvjTXkM2ivZhEp8e+VC9oc0R0QB4MISZIPZoqO9QHrowzhTK", - "Xwbpy3mkJUZUkkNN/C/opq+MSKpHAO4a6i/xIhdVV3Y+w/qiur4p/Gt1j/NZprQYBH3kLFNI/wuWrLdg", - "Rf3VQ5g7P0QvOPSxK+1qRlnTGUxzzKLxYrl5Xb/o2AgQQaTigkQwmSVgQ/Q0J1o52bNkriOJ/WloqY2U", - "giiwLaNQW/HenlbQDSzUg25gQBh0AwcZ/dPsEH7B4oNuYBfiDbJcDlNYGRlRhFrU/eq3CaQpouiphFFp", - "KYYDdTQCli9zKRJ8q42M5hdU9DxNGYv0VWsb47I6pOUMq9kJm/Blk8JtmKV1FDoDTqqRRkIGiYgwSiIX", - "O5VzTYuH4HqMJUFRRizkDF4JbAGOjVklxWoGFx06UjatBl0tTdiGhZk1rH4zAfPahm2kbel3br0WGcDK", - "KMMS4cLN1Uqzp3Lkp8jLAwsyzWIsUD2Oa8WS5SKJKbtuM7pcJGMe0xDpDnVRaMLjmN+M9Cf5M+xlq9Xu", - "dIdRYdGtiTZmcdaebw6kNm+xhZ/1LrdqHsJQyyHbpv82pKRro7x4I52easZnQp0uGH1XQvRqAPL+7qDJ", - "IdwwaMUVvBwm1yamuHz3Lcr6bryLYDvMX0d6zInG0lXj/kcQu2a6VaMFvGFJYBFc5f7Ohyr5wJ0+5AK8", - "q3AtBVq3iut2ZNj7dCGXMRpcoivSdblh/ZT7pGw2r9tj5ok/wFYraU3QOjUq3DK8KlbmBwePHu3tP3i0", - "2wo0Vs/ODTUNhtsmY41bwbYkYe0hcvXEdh8M4H+3WpQx1fiX1GCuqS6o8qj4kxf0ccX1KQJAa2JEfj9W", - "JK0sTtLFilaOcv+gFbRWSCyHFbGnlFuiQyYTAoLvyMCtVyym5pBstYYQpzikauHx2eMb8NGgvEktkLHF", - "6LXFekBqx0Z4orTeOSdCZuMikr/jJkf/gcCGWcOFg9aPVmQ2HsEIHnNvfVZoZ52aUU35LdRtno3jkivF", - "PkfLM1v5LPg3OTDRDZYVq4T+HSoSdUu5Q+rmK9OifWo0h+t5drTCMO8LxvVnQisff+04u0GZmxToXIf4", - "KjbWfAU1VwaPaRsDgYcreiJ8w/UeoBp9sHzw03qNxuXnZCvf61XenrVOurI8rWFEt19uyZFwm471BzKA", - "VnYNFnLF2N3KyfqQwph5ml5fJy4pc+39DDXJNW2QNCo1Rh2SpGrhAgGdkrd1O7PTYT6gF6e+sOt28OhL", - "BI9drIwW+x/ynr9s6XOTrLXxLZ1pY4iGX+o8rnvRjHpl3zNWvT61V1pSrcghuypzuEnhDbqTDY+aZvV4", - "7ltkC2/Sloub45LDunTh65TAhmAI89i3tLPSSprPxph5PzO1OpUup/ongsxqMuvjjYypTOuSvfqDV/Nc", - "RlBQjSyADGA1CHJtd1mlXu19OsXv8hlA8cQS1fKymH2Ucpw9ewxP4F65h4904oaAZdQz7Dz+vJzzDquW", - "D2NVEnrnSPBePEt/VlC0prtVQ85iju7qPPeadJEwE1QtzjVDsD5yggURh5lBQ+AUsAn4czE5xNx9/Aja", - "5sQjdD4jjAgaosOzE8CSBDM81Ud2eYpiOiHhIoyJDZlaMjHDi7+XRyc9E+vpYgTAY00VAMQlwzg8O4F3", - "+DZlbDDo7/YhzxtPCcMpDYbBXn8HMg1oMMAWtyGUHn5am46+h8DJTiLLcR+bJhq0MuVMGuDsDga1FMS4", - "eOu8/Zs0xgrDXlvLdibb+rLbZykSyEkCdvkfu8H+YOdW61n7PNk37QXDmZpxQd8TWOaDWwLhkyY9YUY5", - "dlnoiG1Y4Gww/LWKrb+++fimG8gsSbAWEQ24ClilXDaJMEQijBi5sW8sfuPjPjo3qgW8VS7qWBjNn0Sa", - "JGGksOhP3yMswhmdkytmKbF5ao4FBJQmSFNgE85XRTMztTl9c4WJVI95tKhBNx9uWw8H0kgVwLfOzZzn", - "TUobkjT7qKNJzyBD7s1LQRhmqnjtb/IyXJMFSgWZ0HfekDwIM/Ibjo/zby6bd5W2a3GXsjDOooIBVrMo", - "e596SRIK4hOy/3H+8gWCiwf5mKFZER0FGbQo02QTRRlwHsCU/hV7gsMZMhQVkv9cBTS6Coq8u1tA/TJJ", - "DFHr9YAk/wwJyc00XRr93O/roQy1H6JfP5hRhugqYGkyUvyasKvgYxeVPkypmmXj/NubK+bdcIOufl6B", - "FeoYTN5y71n0DkuX2twCzCLELebEC4RRcUhlWX5MGRaLpiTWPFMjV0Wh4bmPbVbEoj8cDLbW25TtVj18", - "rtJQSwMfl8j67hejaJaaL1O0UsEKTT+YfcsVGTq+AZL6GEcuxPgH71jDO6zQW+IK0N9KDtsfaPTRoG9M", - "THxAjbRDXnNH2lMscEIUERLm9aEFhEZQ/W/nAQIl1aiAVeTtlsBTlwTfLCH2fmPC+Dz1OuDC/gbwD+Yt", - "MmzAvI82NS+OTX63vIjNvUJHOCyHiF2/2PqMqG8B4wabIqUuEdBXxN/7gj/PiJWEC6DVqNk2mTvzo9/P", - "rQTBibSjmMZaCD6HNfXOCVMISpXIvv2vk88gOuptzKdvh8iAMLaFWqRN7ZIbDzVTtLCETubla97PPggP", - "Z5hNiUQdwz//+uNPV2zirz/+tMUm/vrjT7ju27Z0EgyXl0l5O0T/RUjawzGdE7cZqbdA5kQs0N7AJvCF", - "T57n5fKKXbFXRGWCyTzmQ+8LYGIGhHc/DPZDWUYkkgBCSPw4scEIxjbh0Q3cXTag3OiN7i6pSHYHpQ1o", - "ruhwADxblFFFcYx4pkyOJlgHBJUWCzF7DsqT180sS4a39fRFkXfKYG/PLPCWBMaUGfLcO1N5x4yJOufn", - "T7b6CMR9gxUQcAJ6QzGM1QT6P2jSeppkKEqVoACUDW0qZRZqNNIc2zabsNI0ZR1qNtMISJFKtMbqNvND", - "7G5hsvHDzZlvfDaUY5cJs9mI8un79VUhaqVTfrlzdri3DHOb5rUA2dfQJlHHZujLH+JWcsl+LaTfCAEu", - "pSDOqTDi5vnvxjScI84mMQ0V6rm12Mo0udZTRZD7Qg5e2VUj7PZVj5Ius4rtSsBRI9OoFffdDPeoVxS+", - "BRspIrkLXPvBSdahzjGVIdd9S9jSC3FaqmEsi3taxqJ1tp1j+HvOclYK5nmtKFTU992QlcdOnbE6b9gA", - "UTyuEcSvSAhrz1RLbx/uEzZf5Kfosn6vMAJ9W6g52JwUtGmDkA/N75NFKKqBTVPBWZ4Yswm9bOrMOzxo", - "O4Nn4+dEuFttFmqeRxbbMl1ROCPhtdmQTU6/SiI4cfnr714OMPk/b8H97fJ/sPsWimMBq1XK4ol9M3t3", - "umKl5OmG3Y8WwTxABvf+uKgKTiLUwXLBwq3vygO5Ec5QTyZ/j27SWRbHzhA/J0IVWVDL9HT7g5YPWsjJ", - "7ratlEUuXj3vERZyCOYwoGsUSFzSwy8rLZsDM1v5gSZt9CsAlUOMZmH0M87fRHcWFZ//tvvU5iP62+5T", - "k5Hob3uHReHnu0GWwaZI86al13uMfFp4pVWgAWkyqQnXSXt5q40IfDYH7G1EvnyBP6S+NlJfGVwrBb88", - "He8din7VovAb9hPkyOaDNnxy8Wffmci3WdOTxchS4ZqKLd4ml4Ay4DazqC17cf8C5GiOcWX629KGWlzI", - "ldKBQ92T465NGmtSveaRxRuyqLp1bFxKtPNu3px6mIzpNOOZLOemhBzBRBYF1SoE+L7JrwV7bpRgv2Es", - "HWySdWxcQP2B93ckOtcP1BBvW4NtjfDsWm1GeC5cNe2lZ7fCH9JzK+m5BK7V0nOe/u8uxWczyVeTnx2+", - "+QBu375+jxL0fXu2wayNu+TsrdC41gJqkfJyNe8vCuRv3NGfT755udTlkbqf4afcBJxHThIseE2zKPit", - "4cNgs7Rv8yLgfUaxZ+VyJ35hy7y9iPl0/cuLfCT3zMDz9OKKudoob81zyLcoR1SkOJIkJqFCNzMazuAZ", - "hv4bjG9eaeA0fZu/u9waomcQ3ll+CQqTdyQRFMco5Ezy2OSCfTtPkrfD5bf8l6en0Mm8wDCv9t8OkXu/", - "n98xqVuVn1XoXcRYKvTCPhbp6AMXPI5NofO3Gp6l/W3ZBxfFE9Ur5nt8wciNHZBO0NvSO4y3DQ8xHBI+", - "16f0lW5+tzmTtdmL4kgA4EzdGgIVrH2PMGx1bc8TjJ2BNyVNy+cgZhl3/Bqku1xcfJq/+66gMk7Ttuhr", - "lwlYPE+SFTiMOkWtDyRVxDP1d6kiIkw5M4vdTciNOjg0/1D42hTfqlQfMRmKfaCyT5u9oApMiUGX2Nj8", - "a54kgSmFkmBfouLPf1ZTH3BZH9MnU3o784Nn3OZVTJXYl57F1DiHzZANWSC8ytsr0+C7l1xcKvGvjIab", - "d0WUVkGhHASLxgs42yJH+/16EwAHWewM+J3dl/eOuG+Nd8Smdv/u70iBH9/5LQm5gHqQ0tVnuT/BWyWN", - "o3TdO1AQoii00HVa7+Xp6VbTpTElBRuvjPihDts4yu+ep0CNjPt3W0x5KJxvYJWxUF8I1aijO521Uj9n", - "zDM9+lJaS6h9IBdSkcQo7JMshodtELVu8wPgcm2HLqJKQpbjLpisSnn9r9iYTDQ/TInQc+vukDar0D18", - "au25wvn1PTN38NvQayHTJahyWDVBbanEukty6dOd8rycn7ykp6CoVmtLSNSJ6bUpNoPmEsX6x9ZKTdcU", - "nvjS2Q8+/WblpVV8r1oNzubI/D1QuJMaWXNll+4dWXtGypfF0R84aB9Z4+kqNs/TH1ze1oD6IRPfS5kY", - "HD35bjpTgUPguNJW+fLLv7a8y/YH8+NknbtQ4XB26VKEfxus1GYUXjeN2+C9uJR2TxExT3o3fyd5nvT5", - "nj7b0IBzWwDTSdnx6ecCJpn894bdXz7GpQzHW0W4bPRuuefy38zd2jTns2tw4dpleNyXa24wze0EkheX", - "VVtRLk6zUqF1RUegUpLrllf56ZZLN5nsfLmCWhQLyKvE9K9YXhbHZQdER2cXXVsEugtlps0ItqhKH/mr", - "F0mEBXEljK6Y4ijEcZjFWBGUl/Expbdkg1v3Vam01Z3dt2ISz0Hn9YtkXqfmPukYfpyA0ysXwgGMK1VQ", - "bowttcWUNxJZapnZLeJK3Q5+hOC1iCotAatN2n3TvI/OszTlQkmkbjjUsZTgy4cki2MeLYYo78eQKX1k", - "SZytWWPzz5MI6obovqeVXPylAVzPVJBeylMgHZF5smNhbMSj5Sz/DYn8c/no7sJj66JD97a1AUprqZ5H", - "dY8oT7xvc8Fr2Fp4uSFaZXz3FTrJiw+EmVQ8ceOeHKMOzhTvTQnTwC3y/KeCz2lULxf3jdR4OsXvaJIl", - "eYHUZ4+h3KQwoR5QeBgCjRxOkXchIZGEyI+tW9aDWi4FZc/i03Lefzki5qhpo0z5FWOmi+SE+oi1jOmQ", - "XHGOYiymZOu7eZlo71rxMPHkuPYs8R5Ge88d9hVyRsv47nYqbUtN8y5iu3Nzx2Yjuy+/HS2slL/tHj4v", - "nOdiZlNI+beFgoPNsYRNh5Jf3mOrnda25jWwmQH0iD6Eec5DHKOIzEnMU6h4aNoG3SATsa3fNtze1mpa", - "rBW54cHgYBB8fPPx/wcAAP//PPEgqyLOAAA=", + "H4sIAAAAAAAC/+x9a3PbuJLoX0Hx7qmV9+hl+RFbW1O3HDvJ+Gyc+Maxz90zylUgEpIwJgEGAGUrqXyd", + "HzA/cX7JLbz4BCUqDyXeZGqqYpsk0Gh0N/qF7g+eT6OYEkQE94YfPO7PUQTVjydCQH9+Q8MkQq/QuwRx", + "If8cMxojJjBSL0U0IWIcQzGXvwWI+wzHAlPiDb1LKObgbo4YAgs1CuBzmoQBmCCgvkOB1/bQPYziEHlD", + "rxcR0QuggF7bE8tY/okLhsnM+9j2GIIBJeFSTzOFSSi84RSGHLVL017IoQHkQH7SUd+k400oDREk3kc1", + "4rsEMxR4w9/yy3iTvkwnvyNfyMlPFhCHcBKiM7TAPqqiwU8YQ0SMA4YXiFVRcaqfh0swoQkJgH4PtEgS", + "hgBPAaEE7RSQQRY4wBIT8hU5tTcULEEOzAQKpjEOHDtweg70Y3B+BlpzdF+cZPBocuTVD0lghKqD/ppE", + "kHQkciVYdnz1bn7s5/uukTGNomQ8YzSJqyOfv7y4uAbqISBJNEEsP+LRIB0PE4FmiMkBYx+PYRAwxLl7", + "/fZhHrZ+v98fwsGw3+/2XVAuEAkoq0WpfuxG6W4/QCuGbIRSM34FpS9uzs/OT8ApZTFlUH1bmalE2Hn0", + "5NeVJ5virrjo/3GCw8BB9VQCJlAwhqK6KPURMO9gSoDAEeICRrHX9qaURfIjL4ACdeSTJqTuMwTXTCff", + "aDRZlegTjdNxxOtGt68ATECEwxBz5FMS8PwcmIjD/frF5EgXMUYdsuKJ/DOIEOdwhkBLCjApRQngAoqE", + "A8zBFOIQBTtNUOaiYb2Y3+kE4AARgae4yGneRL7QgRN/d7Dn5OIIztA4wDNzJhSHP1N/B3QK5DgCqLfd", + "C5Ekv2y2DjUlQ9PqfE+VEFWTMDRFDBH/s6eLGV0gAokW9v+m5vX+Vy87LHvmpOwpZF5mr39se+8SlKBx", + "TDnWEFZkiHkiyUihGqgv3DCrR6v2OkdRXEC2mj/UG1+AEzV8jXBzpV8tSyYleMwwBc6uFUBPFogIlxQi", + "wjworvg5nYEQEwTMGwa/U8qAnOCXkM52vC+ztraXobTK0BLuTxBI+g81o8lnbQ+RJJLIDOksj805gkxM", + "UAGZNQeEGSiDrhb9lwWWKO7BBHI0Xi0VLjEhKADyTcOs+k2QcKUHVpavOOMWi/ECMe7kIwXWf2EBzBu1", + "Q4XUv53iEI3nkM81xDAIFA/C8LKwEocuVFAuYSwFmx1QndEcCAqufj0ZHBwCM4EDh5wmzNcQVFeS+1oO", + "r98FArIJDEMnbdST2+bnbpVC3BRwlTJG3XmSUqAlTC29PLObcvi2Fyd8rn9S8lhCpc4zKQYkeYXy5zeO", + "RZ8qIaF18FqLxK1hvYz1ZoNZSCVOlyAh+F1SUF+74Fxq4gJI4Y8DFLQBVA+kGIaJoJ0ZIohJOQWmjEZA", + "zBHIqZighbqzbhuMpNbVkTpmBw46/X6nP/KKSmK435nFiUQFFAIxCeD/+w123p90/tXvHL/Jfhx3O2/+", + "/m8uAmiq90pyknCadbYs77eBBTavDJcBXa0or9A1XVJEb9+55P1Nd+/0vHrAa/gD6t8i1sW0F+IJg2zZ", + "IzNM7ochFIiL4mpWv7t2fQq2FQsjM7n0DZdWUv0VubVCeoeYLyVliCSB8LYUlljwNoDSelRCBsjT7D+B", + "D4mkWX2wUwYQCcAdFnMA1XtFDETLDoxxB2tQvbYXwfvniMyk+X64V6FHSYwt80PnzX/YP+38bydJsiRE", + "DmJ8RROByQyox/r0nWMOMhiwQNHa49ZiNwmVihVhcq4/200hgYzBpXvXLHCrdo8LKXxqt08zkGN9Z9bA", + "5sAYbepAgMp9otb77PK6J1kyhpyLOaPJbJ7fld+sPHiTw0WNNmAX2fYCzG/HmI4nsQsmzG/Bee8lkNIK", + "hDjCIpNOu/3+xeMeH3nylwP7y04XnGm/igJfLp4yIzT5HDKkju4AUAJOL68BDEPqG2NoKjWsKZ4lDAXd", + "kjWsRndRCyKLzziHn5AFZpREUhdaQIYl8xRs/A/ei5dnT8ZPXtx4Q7mTQeIbg/ny5avX3tDb6/f7nuuo", + "kzuxhhifXV6fqhXL9+dUxGEyG3P8HhW8U97es8deGfCTdL0gQhFlWh81Y4DWvCgO9HENQnyLwEiOpzdt", + "91lZUA/UVBWkzZcxYgvMXXbmr+kzud8JR3ne1MxQJAmO2AKxdK/V5ndzZ70f0iTo5KZse+9QpMg6A9Tx", + "ktvWa3QKrBHvMIwxQbXyvf29yOQ7ym5DCoPO7hcWyQQJOXZ1iS/0g+JmGgJA6f577YqeT4I7HIj5OKB3", + "RILskD3mCUhfTgXQvVwJDP/648+bi0wB2X02iY002h0cfKY0KskfObTTuEgXksTuZVzH7kXcXPz1x592", + "Jd92EYhI+gwKQkfb68Wl/HOOxByx3KlkN1j+SWuH6nNg6SU3fcEBkPefVwQnXSAWwqVDEO72HZLwnwwL", + "xV/mOyBPNCA/XiMG5Wj28KoKwr5bEjqAcsD0WPK3kctNIEkB2R1cmB8HTWXzwo+tMWVAGpTBeaGc4FKF", + "X2AmEhhKOikcc06fuI62ONQCHczJqydm/1N6gKLoQm2qnumRVeilqqy4NTIt5es1sjWRJ5djM7Xy/IQL", + "GuXcm6BVMuBw0dQr7tiChp0ACqjkccNDQ4NbddpHSz2U3pQ60hzPJg6vgKRATMAMz+BkKYoKzm6/uvVu", + "RNvxXaiuC2hp8kDBWFBHnMZSy/mZxKN9t4nfUIW/xoKOF1PsGDmVVJnFijnwS9EzQ7RyiE7sYxNNa4O7", + "OZayjQOLBHWg3VzkFe/uiHSABG4IztIJ0mHTIeWRrrwTaogWZTkgsHI0gclyB0Bwc9EFr1No/50DAgVe", + "IBvhm0MOJggRkKgzEQVqfhW3zAOQcGkhYVH+3OjsOhi4o+wLap51gVTgIkjAHQ5D5Z+IoMC+cm5McGk9", + "yqmsN0rOJAUAydS8EclTlomqlkX+6vDLKzTDXLBS8AW0Xj093dvbOy4L6cFBp7/b2T14vdsf9uX//2oe", + "p/ny8U7XWCdFeWHcRXmJcnp9fjYwJ0JxHvF+Hx4f3d9DcXyI7/jx+2jCZr/vwa1ERN3i6Szzc4FWwhHr", + "WNEnqcrl3co5kWq8V5/slNooGGvd4KuOH7261/LNrxG+dYUujON88wBrWQiuDX7kFldZj/yr1A8yys8Z", + "ZMbH6GOnN/UM89vHDMFbqco7zld5PPOxPnfcDoZEKq+TJUD3Uq9FAWCUiinXRlpRTdndf7R/tHe4f9Tv", + "O2KlVSKmPh778lRpBIC0DEO4lMap/Aa0lHYdgElIJ0XiPdg7PHrUP94dNIVD66bN8JBqUfYr0DIY+bvN", + "gLFPCkANBo8O9/b2+oeHg/1GUBkFrxFQVhksqA6P9h7t7x4N9hthwaXrP7Gx63IsLnAQ6Ukch1hbNh0e", + "Ix9PsQ9U9BvID0ArUscSStXsIk9OYDBmRg10ngcC4tCBhpyrRU9m3gQteaZHSShwHCL9TG1II01XrfxM", + "jeRyy2FCEBunof0NRjIR/7XuCLuW9BWlogRoksxmOqySoe4Cc6VZZAoRRmEw1By6Vs6p3cwAe1NHB2YN", + "DanhOb1DrBOiBQrzRKCPIwlsRBkCKZ3oTSusCpMFDHEwxiROnCRRi8qnCVP6pR4UwAlNhNIl9YblJ1Fx", + "CmUjTKW4bhYmyxyDlamfXV5v6m2JGZ3i0LGMhRzMPDVHuvVDPN/vX3V2/49yPrwk4VLLAUyA+iaiAeqW", + "0rTU+42Xd1kHU5ojB/LQVdYE7WsOn1Rq7VqMcCDm0iKFBEwQMMek9qQpP2U2SSbgj10Cc8pghCbJdIrY", + "OHJYWk/lc6Bf0IY/JuDicVFoSuHcVN26LGyO0rem0MdkttMY+w5LrrSMdg6bb9zb9QrpWHJd6FZuFTPv", + "mOhtF7xIsxLBs8trDtJZug4Tr2GU5HK+5NI40SPqTAxM8paZIs7GYvgy+9DYsA5hHDkFkGUE0FrM4kSx", + "4dWrzvnLm14UoEW7AJN8eDenIZJw7+R0q4UN4GYhnYKfe1GnImvC4E0ZKIerlIMbIynHrw7sCCpgOOYh", + "FQ5oXsuHQD0ErZunOnAnIWiDuLCV8u85LBTo+9DJMVIi1U17pSYs29oFBl/r9oj0sZVfXmFSF6v8imCo", + "c5iL9JzlAtmNp7fFjaa3a7nXDOKa99yGOkonZ+SwXU4vzrRl5lMiICaIgQgJaDKmc+FEFdX22l5HKgMB", + "RBElgE6n/7k6wFjju0nJZZX1f1pJu/wqln9NapEUcuECBSCCBE8RFya1qDAzn8PBweFQJzUGaLp/cNjt", + "dt1udcGWMcWunLIn6bNmW9HTQalONmaXzz9vH75C4LTJWj54lyevf/WGXi/hrBdSH4Y9PsFkmPs9/TV7", + "oH7Qv04wcQZcG+XB4mkl/7WwvbE8s/Tfh3IlBPkpQVKlJa71TbpP8heSNEP8HgXAmYYi4AxI/VtR3Ofl", + "m3xG5mh2kUDkMkbzYYIG2aP4/Wpz2ypG6h0zZ0IEDrPE2qqh/Ump0XxlplklyyxGJM0tC0P9k0/JQnKF", + "K9GsIMDts8pm3FF2i8lsHGAHdf5TPwQBZsgXKo6/noe8Hozj9aToVv5SmdY0adakzDhOl28uyT/F4Vqc", + "/eXsH+/+L7989Pvuu+c3N/+9ePaPsxf4v2/Cy5efFedfnS31TVOeVsbUlJexkOrUlDwuoPAdis+cclGD", + "NfMECAoi+XEXnCoDbTgiHfAcC8RgOAQjD8a4a5DZ9Wk08kAL3UNf6K8AJUAOBeYIBojtyI8vda6D/PiD", + "tQE/lscIlgRG2AfMIDmNofNkEtAIYrIzIiNixgJ2IVwFbeRPAfBhLBKG5I5IXTNcggmD0tw0ZnQ2eRt8", + "gHH8cWdElCWK7gWTK4ghE2lqpZ1BbbSBSgeFzOsoAAsYJogbS3ZE0vNDmeZyEAHZDIlu6kJUjppSYKYG", + "KU4zgzJRiC0f9duOfQTyPbmRIeYCEZB6JTBXxAtaNjPgqF9g/6P+0fr4Y0pDK8hPUXf1WqElygb8oQlY", + "Ta2F8XguRLz+nqCSN5pHwK+vX19KNMh/r4AdKMNFusXaGINxHGLEdVRNhEonMckYO54rcqZ3t+GCXuuX", + "5WchX7+OJ2pi8Pr5FRCIRZho+d3yJTqn2JfrU/EdzHkiSRFDcHJ68WSn2+BepMJtCv+KfXydrrAURrDO", + "raqFqb7InOYSv21wftaW6pTh0EzRUnHTp5SBUAuYjK+H4JqjYhaD2iod4tE7GS4zD5mW6iNvx44YlyXF", + "ELxK9TuYgpKmfGfEYIfM+FINOyL/lIShg7qV0dtFWFW42tgvRrSpEC4UwDi91VFcLwpWs78D44rnKSn7", + "Hjfj7bzTUk7mJo1s77+6BrK3qS25afpsMRMol/mVZtB+29TXT0lktTv07PJapYtCPuYExnxORX1yBgT2", + "HYDuMRe8mjjaKJ2gmjhbPJ50SuyKbKwvmQLLEkJUZkR5GV88ufVb5hp8f4m1K1NhPzef1ShoXymdtVYg", + "uFJBi7JB//nLJqZ+FXAKKaYuYZA/x2wi2CdnlbY97EiCOeEczwgKwPlldtUqc3jY4UtrOh50dw+Purv9", + "fne338T9E0F/xdwXJ6fNJ+8PtEE8hJOhHwzR9DPcT4awtcIBwzu45GBkVcKRp3XQnPKZY1ujNjYK7VWT", + "dz8tV7d8CK7Lxt0k+7aRvF91B/qqePu5sV5x8K/PuiiNmh7DV+pl+9V4E8coAj5NwoD8uwATyXnaFECB", + "sVg4EtnFcsWs1+SW0DtSXLr2j0n+fZcgtgQ3FxcFbypDU3PHtsHCaRzX7gONN9qGwRr1bi00uWTrbSRY", + "lyVh7gT64unUedePzevQVNfABZSpf84wKSYa3XLvV6ypZLwHaDFOEpeiIx/ZDM3r6/OzwoZDeLh71D86", + "7hxNdg87+0F/twN39w47gwPYn+75j/ZqqlE0T5P49MyHIofWZ0QrxCtHmE5iD4aSh9LUhUkiQHozSDLn", + "qdQYQU4P1fm/yjZ9pVVSOYI6XX35JFymqurKjy+hZFT7bax+W/3F1TwRUg1S3/B5IoD8TYEsl2BU/dVD", + "aJ4fghdUfWMgbcuDsmQz6NchCSbL6utl+6JlMkAY4oIyFKjJjAAbgqep0ErFnhFzLY7Mj1qWmkwplQW2", + "ow1qo96b3fLansG61/Y0Cr22ZzEjf9QrVD8p4L22ZwBxJlnm6cblKEYwVDIsS8JIBA7xe81yEnTMBfa1", + "iQXVbtaxnbk+hIKxPkLrQjk6sm+O2fQjy9U3F6Cl7h38HRgLTP62k4Z98iy0PzjePz58NDg+bJS1mAG4", + "XhqfqryTKnBrRbMfJ2Nbladm6aeX1+rwkQcbTyKdJmnWntluUnD4UtvDBGRlfrLJj7vH+WTNgCaTMOdp", + "MNnaKiNQb5gzHzkVHDVxjnc4XODplLx7798Ofmc42r0/5IOJ0zhKJ3Jrkud571jF7EKTjr7f6c6nUwTF", + "eG3K6SvE1QrAFRJA0U9HCix5oqbpIobkbGKqwbiTsPb39vaOHh0MGtGVgS7HOGNl/1WhvDAQ5FhMvQla", + "r66uQC9HcHpMm0MXM8Tl4vQtCiefgVHS7+8h0C+k10nbY89FJTUKS0Y1ZuxFVIvyG6OxmEUZpKusl1Sb", + "qXC5E9t7e/1H+wdHB83Y2Fg8Y3a/WsKY90y0mCEf4UVh51vKo/r65BLI0dkU+kUNf3ewt39w+OhoI6jE", + "RlAJBgmPsBAbAXb06PBgf2+w2yx32uU1NbcCCgxblF0OpnMQhWM3HKioit523Wnh0hKrqXYrs/uydMFy", + "btgmyaDZTTDM1ag4l4cIWlKJyiukudtMO038DG4RKeepq7on1cWmeZqr0zIvoZifkymtusU3MfhMsosN", + "QsRS8eGqClKACEaBlV2p5Wd0KZU+E3IEggQZzGndiEGDcKhDAzEUc6Wsqg8xmRUThysTNjHDNAyr7/2p", + "ec2LTTxG3J2g8ZolClfaocsBzFI1GnmnMR+7rYrqwAzNkhAyUM5FXgEyX0YhJrdNRufLaEJD7AP5Qdmc", + "n9IwpHdj+Yj/otay02h18oNxFpUsmecaOBOT1htSmjdbwi9ylTulLBd18vf09z1VVrWJA86ZrftUGm86", + "Xfea4PscoRcv0ewP+nVJTTWDFtKZqqnem8p2Q7IujrdZ2CfpDX9HSExHa0oWbFEPLqzXtVoV1VqVwlXV", + "BEDL+vTsJaUiXnOXhRodxJ+l7q4oOfkZyu0iWqHU1mDroonmdHRwdHy8t39wPNhIR7HBhprgY13AwULQ", + "48gvFdMo6U0HffXfRkDpcIMbpJqQQxGgQmGMTwbo4wr2yS4x1Fi+qwovZztpTe2iEtxMzVyhsZwU1J5c", + "faQWmk6Rct6MNd46GTClpJpGMPgwhj4WS4cVBu9UngFIXykl4zcYvQSsA6VmbACnQlrrC8R4Msluo7Xs", + "5OA/tHVVooWjxhcveTKps+RelmfVdpxOzAlKXoIGRrqmCFcU+i5FJriDvOBZlz/7AgXtXP2rcghGv9G8", + "vKel9bTCZxZcdl0ocVfzzG9/aTtzmn9BUS1jfNUxVs+C8lRWWT9NnNyOU9FxS8Vfn8VQkg/mHPy0r8aT", + "/JXolXfOC/enGxcOq06rD6LNwc0Fwzf5sHzJU5GVgcFgLhu7XdhZF1HoUEVdBZHINhYo3QHFukC0uegD", + "ci+DFopisbTJ7NbI29ksdHKSDuikqS+cftQ//hIJ0NcrM57/h9SkyUer7CRr41SVPa1NM3RrnWflTBBt", + "Xpk7+cXMhdJNYy5W1EFf1f1Ct6FQtpNJ8Z0l5TtJG3S8qLOWM86xBc5ty4t1RuBK11RuZTlI6vdGhyo/", + "sz0I5rYvyCeizFgy63NmdbhH2pKdctEGfeWTYWUaGQRpxEoUpNZu1aRenUFxAe/TGZThCTko1RbT68jV", + "6Xz2WF3jfmUv7+OpHUKBUa4S9/jz+qZYqqpuxqpGKjYY7mQ8I39WSLQ63ioRZzZHe3WvFim6kJ8wLJZX", + "8kAweV4IMsROEk2G6qRQi1B/ziZXeeMfPyprc+pQOp8hghj2wcnluaKSCBI4k1t2cwFCPEX+0g+RSfut", + "hEnVrfWXp+cdfV/B5rmprCssFEJsQaeTy3NVS8aUPff63UFX1SqlMSIwxt7Q2+vuqmo5Eg1qiT11HUz9", + "aHw6kg/VSXYemBP3sX5FopbHlHCNnEG/XyqjD7N6Hb3fuXZW6OO1sW6nO4ZUUxcq2axWEzDgf2x7+/3d", + "jeBZW2LDNe01gYmYU4bfIwXmwYZI+KRJz4k2jm0lVWRezGjWG/5WpNbf3nx80/Z4EkVQqogaXRmuYsrr", + "VBjEAQQE3Zl7gr/TSRdcadNC1dvIejFpyx8FUiRBICDrzt4DyPw5XqARMZJYl0uBTF2KiICUwDolvUhm", + "emq9+5qFERePabAsYTcdrieHU9pIEcEb9xdIa//FNY0GXNJRlxjiPnXWVkIEEpFVrNG1hW6RigdO8b0z", + "rVylyrodx2fpM9uRoijbpbqLiR8mQXYAFjsBOK8rc+Qz5FKy/3H18gVQjKd6CqjXsgxfVQUSEyk2QZCo", + "k0dRSndEnkB/DrREVQXsRh4ORl5WO35HSb+EIy3UOh0lkn9RTTX0NG0c/NLtyqG0tB+C3z7oUYZg5JE4", + "Ggt6i8jI+9gGuQczLObJJH32ZkScC66x1a8KuAItTck79k6mXGGOqTUXQBIAaignXAIIsk3K6/ITTCBb", + "1jVioImozznQV1bNa9l9qsN+f2e9T9ks1XHOFV6U2sDHilgffDGJZqR5VaLlmi5J+UHMfeRAy/EtiNTH", + "MLDXZH6eHWvODqP05k4F9b3RHHofcPBRk2+IdI5bSbSr3hxWtMeQwQgJxLia10UWKr0Py99tBEgZqdoE", + "LBJvO4eesib4pkLY+7VNT9L2IYoW9rdAf2rerEqUmvd4W/PCUNcoTRuxPShyVJtlCbHtVlufIfE9UFx/", + "W6LUFrP7hvT7UOjnGTKacIa0kjTroYV1P7rj3IIhGHEzin5ZKsFXCqbOFSICqHZbvGv+tfqZyvB9G9LZ", + "2yHQKAxNszFuypOlzkN5KBpcqo909Yb0O1PUxJ9DMkMctPT5+dcff9qGSX/98adpmPTXH38qdu+Z9n9q", + "uLTV19sh+C+E4g4M8QLZxaisNbRAbAn2+qYIvXrkKJHCR2REXiGRMMLTnA+5LoUTPaC6u0rUejBJEAdc", + "oVAVL56aZATtm3DYBpaXNSq3ytHtiolkVpBbgDwVLQ2oyBYmWGAYApoIXWdQwaEuRmSA6DV7+cnLbpaK", + "4229fBHoXmjq7WgANxQwulWeg+909zg9JmhdXT3Z6QKl7muqUAknym7IhjGWQPenTFovk7REKQoUhWUt", + "m3LV8WqdNGfmnW14aeoq59W7aZgq842kxWoX81PtbuCycePNum9cPpQzW8253ony6et1ddJrZFN+uX22", + "tFfFuSlVnqHsW1iToGWqzKbFJAr10L8V0W9FAOfK6KdSGFBdwmJrFs4pJdMQ+wJ0LCymu1pq9RQJ5KGI", + "g1cGagDtuspZ0vmjoldIOKo9NEoN6rdzepS74m9wjGSZ3Bmt/TxJ1pHOGeY+ld/mqKXjwzjXh59nfJqn", + "onW+nTP19/TIWamYp/0OQdajfkteHjN1QspnwxaE4llJIH5DQVgqtZC7+/CQqPk63UXbuWKFE+j7Is3+", + "9rSgbTuEXGT+kDxCQQltUgrO0+LOdeRlyj9/xY02MzgWfoWY5WoNqL7iny1Lfwr8OfJv9YJMg5VVGsG5", + "7cHy9fUAXcN6g9PfgP/zuG9gOGa4WmUsnpu6D1/PViy07d5y+NEQmAPJKrw/sY5UXVIB8iXxd36oCORW", + "ToZyQ5QHxEmXSRhaR/wCMZFV8s7L094HqR800JMtt63URa5fPe8g4lOVzKFRV6uQ2MK9X1Zb1huml/KT", + "TJrYVwpVljDqldHP2H+d3QnSinp/Gzw1NfX+Nniqq+r9be9E19Xb+WrE0t+WaN629vqAiU8qr7iINCWa", + "dHndddpe+tZWFD5Tx3wTlS8F8KfW10Try6NrpeKXlpT/iqqfqdT9beIEKbG5sK0e2fyzH0zl267ryVBk", + "rvlawRdviktQllXHNq2bHl6CHE4pLi9/G/pQM4ZcqR1Y0j0/a5vC57pceZpZvCWPqoVj61qimXf77tST", + "aIJnCU14vr6yqnOPeNYUtCCAH5r+mh3PtRrsd0yl/W0eHVtXUH/S/VdSncsbqoW36SO6Rnm2b21Hec5C", + "Nc21ZwvhT+25kfacQ9dq7Tmtpfk11Wc9yTfTny29uRBu7r7+iBr0Q7u2QYyPOxfsLci4xgpqVj929dlv", + "aONbBPrTybevl9o6Ug8z/ZTqhPPAaoLZWVOvCn5v9NDfruzbvgr4kEnsWb5ll1vZ0ncvQjpbf/MiHcle", + "M3BcvRgR29/rrb4O+RakhAoEBRyFyBfgbo79ubqGIf+mxte3NGAcv03vXe4MwTOV3pm/Caomb3HEMAxV", + "UWga6nrmbxdR9HZYvct/c3GhPtI3MPSt/bdDYO/vpzzG5Vv5axVyFSHkArwwl0VacsMZDUNdgPatxGdu", + "fTvmwkV2RXVEXJcvCLozA+IpeJu7h/G25iKGJcLncpe+Eee36+t/67UICphCnO69hkhQcwlDYs19BWO3", + "7yxJ0/A6iAbjK98GqQDznM7Se98FUoZx3JR8DZiKihdRtIKGQStX85yLgCbi71wEiOmWnIa664gbtKCv", + "fxHwVjeQLHTQ0lX2XagyV5udqPJ0m1xbnF//togiT7fziqCr2P7nX6spD1i1x+TO5O7O/DwzNrkVUxT2", + "uWsxpZPDdHlQVSCcxtsr/cIPr7nYdhjfmAy3H4rIQYFVlwwSTJZqb7M+Iw/rToDayGxl6rwz63LyiH1W", + "yyOmPckPzyMZffzgXOJTpnoac9tj7OEkb+Usjhy7t1RTo6xZUNtavTcXFzt1TKPb4tayDPtpDps8yh/+", + "TFF9nh4et+gWhzBdwCpnoWQIUWujW5u10ANuQhM5eqWspep9wJdcoEgb7NMkVBfbVNa6qQ8A870d2gAL", + "rqoct5XLKlfXf0QmaCrPwxgxObf8XJXNymwPl1l7JWDKvpeaB78Pu1ZVulSmHBR1WCs1UIhjW+TSZTul", + "dTk/GaSnylAt9pbgoBXiW90wDSw4COUPOystXd144ktXP/h0zkpbq7hutWqaTYn5R5Bw5yWxZlsHPjix", + "9gzlmcXKH7XRbrHG18o1tmHvPYu7XA++7ohcIMHkO5Ah4NMwVOXetf7eixn1e6ovmB/jQDcIU8ApgVf/", + "OFIznl5eq/d0ae72iMhfqp2pyoDaBlfnvZdrfH+6J+H/YD1HL3AVW7g3/KdbZ/NQQC0P8RoWpfEqTZzG", + "PxVx02r0p9n6IM1WFYtNV9OaMegrpZibZrJuE9V0YOp90D+cr4voC+jPb2wV/+9D2zVFv9dNYxf4IJjS", + "rClA+tb99nmSpnXZH+jNKok4uwSlxORzE9yngO738KNR95dPQ8vjcaMktK3ylq1o8d3w1rZPPgODvVGR", + "x8dDYXNNaXYlqr543vvE8v2jVtpmti+QamaWqpa2rVU7311NF9BMfUhZP4+0kVN3RNLOVbaAp7Su2ta0", + "AgHmt3oEYz11gbvBmLbzTJexEREU+DD0kxAKBNJOW7o7Hq+xvl7lus99NX7LJnFsdNpijKetpB6SyeGm", + "CbV7+V5ViuKMOrUy/fvGvLON5G9zmG2Q+m1X8DNLtkHidw5ZTTpj6Ne74CqJY8oEB+KOqlazXKXbqDqo", + "ExoshyD9jgDdncyIONNWyrSIQIFq7SO/vSi0y8gNYL+MGerENFaiI9C36gyOtXpUbcRR02sj1Y++XgZ7", + "WXVob9q+IwdLcT+KawRpbwzTrkHi1uDLDtGoKYOrF1HaH8RPuKCRHff8DLRgImhnhohEbtaKI2Z0gYNy", + "R8fvpA3bBbzHURKlPYyfPVYdYZnOxlK9wVUuoKUpdO8jFHCVnLWzYcu2arc2sxef1pbiywkxK01rdcpv", + "eK0hqx8qt1jqmJbIBaUghGyGdn6Yy8OG17K7w+dnpZvDD/BCxsJSX6ZnNLyC0cykbWhpfo3rF6m7Y7uX", + "L26+HyssV2LxAd4AXqRqZt2tj++LBPvbOxK2fdvj5gF77aS1tSihTQ8gR3QRzHPqwxAEaIFCGqumpPpd", + "r+0lLDQtFoe9njTTQmnIDY/6R33v45uP/z8AAP//VH7pVYnYAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 9f0bcd06..9aede871 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -22,6 +22,7 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" "go.opentelemetry.io/otel" ) @@ -165,6 +166,23 @@ func ProvideResourceManager(ctx context.Context, cfg *config.Config, p *paths.Pa return mgr, nil } +// ProvideVMMetricsManager provides the VM metrics manager for utilization tracking +func ProvideVMMetricsManager(instanceManager instances.Manager) (*vm_metrics.Manager, error) { + mgr := vm_metrics.NewManager() + + // Adapt instance manager to vm_metrics.InstanceSource interface + adapter := vm_metrics.NewInstanceManagerAdapter(instanceManager) + mgr.SetInstanceSource(adapter) + + // Initialize OTel metrics + meter := otel.GetMeterProvider().Meter("hypeman") + if err := mgr.InitializeOTel(meter); err != nil { + return nil, fmt.Errorf("initialize vm metrics: %w", err) + } + + return mgr, nil +} + // ProvideIngressManager provides the ingress manager func ProvideIngressManager(p *paths.Paths, cfg *config.Config, instanceManager instances.Manager) (ingress.Manager, error) { // Parse DNS provider - fail if invalid diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 3bd9ee2d..39e7e555 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -117,6 +117,19 @@ type VolumeLister interface { TotalVolumeBytes(ctx context.Context) (int64, error) } +// InstanceUtilizationInfo contains the minimal info needed to collect VM utilization metrics. +// Used by vm_metrics package via adapter. +type InstanceUtilizationInfo struct { + ID string + Name string + HypervisorPID *int // PID of the hypervisor process + TAPDevice string // Name of the TAP device (e.g., "hype-01234567") + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) +} + // Manager coordinates resource discovery and allocation tracking. type Manager struct { cfg *config.Config diff --git a/lib/vm_metrics/README.md b/lib/vm_metrics/README.md new file mode 100644 index 00000000..3257b84a --- /dev/null +++ b/lib/vm_metrics/README.md @@ -0,0 +1,130 @@ +# VM Metrics + +This package provides real-time resource utilization metrics for VMs managed by Hypeman. + +## Overview + +VM metrics are collected from the **host's perspective** by reading: +- `/proc//stat` - CPU time (user + system) for the hypervisor process +- `/proc//statm` - Memory usage (RSS and VMS) for the hypervisor process +- `/sys/class/net//statistics/` - Network I/O from TAP interfaces + +This approach works for both Cloud Hypervisor and QEMU without requiring any in-guest agents. + +## Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `hypeman_vm_cpu_seconds_total` | Counter | Total CPU time consumed by VM | +| `hypeman_vm_allocated_vcpus` | Gauge | Number of vCPUs allocated | +| `hypeman_vm_memory_rss_bytes` | Gauge | Resident Set Size (physical memory) | +| `hypeman_vm_memory_vms_bytes` | Gauge | Virtual Memory Size | +| `hypeman_vm_allocated_memory_bytes` | Gauge | Total allocated memory | +| `hypeman_vm_network_rx_bytes_total` | Counter | Network bytes received | +| `hypeman_vm_network_tx_bytes_total` | Counter | Network bytes transmitted | +| `hypeman_vm_memory_utilization_ratio` | Gauge | RSS / allocated memory | + +All metrics include `instance_id` and `instance_name` labels. + +## API Endpoint + +```bash +GET /instances/{id}/stats +``` + +Returns current utilization for a specific instance: + +```json +{ + "instance_id": "abc123", + "instance_name": "my-vm", + "cpu_seconds": 42.5, + "memory_rss_bytes": 536870912, + "memory_vms_bytes": 4294967296, + "network_rx_bytes": 1048576, + "network_tx_bytes": 524288, + "allocated_vcpus": 2, + "allocated_memory_bytes": 4294967296, + "memory_utilization_ratio": 0.125 +} +``` + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Host │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ /proc/ │ │ /proc/ │ │ /sys/class/ │ │ +│ │ /stat │ │ /statm │ │ net// │ │ +│ │ (CPU time) │ │ (memory) │ │ statistics/ │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ └───────────────────┼───────────────────┘ │ +│ │ │ +│ ┌────────▼────────┐ │ +│ │ vm_metrics │ │ +│ │ Manager │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ┌──────────────┼──────────────┐ │ +│ │ │ │ │ +│ ┌──────▼──────┐ ┌─────▼─────┐ ┌─────▼─────┐ │ +│ │ OTel/OTLP │ │ REST API │ │ Grafana │ │ +│ │ Exporter │ │ /stats │ │ Dashboard │ │ +│ └─────────────┘ └───────────┘ └───────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Limitations + +These metrics measure the **hypervisor process**, not the guest OS: + +- **CPU**: Time spent by the hypervisor process, not guest CPU utilization +- **Memory RSS**: Physical memory used by hypervisor, closely correlates with guest memory +- **Memory VMS**: Virtual address space of hypervisor process +- **Network**: Bytes through TAP interface (accurate for guest traffic) + +For detailed in-guest metrics (per-process CPU, filesystem usage, etc.), +consider running an exporter like Prometheus Node Exporter inside the guest. + +## Usage + +```go +// Create manager +mgr := vm_metrics.NewManager() + +// Set instance source (implements InstanceSource interface) +mgr.SetInstanceSource(instanceManager) + +// Initialize OTel metrics (optional) +meter := otel.GetMeterProvider().Meter("hypeman") +if err := mgr.InitializeOTel(meter); err != nil { + return err +} + +// Get stats for a specific instance +info := vm_metrics.BuildInstanceInfo( + inst.Id, + inst.Name, + inst.HypervisorPID, + inst.NetworkEnabled, + inst.Vcpus, + inst.Size + inst.HotplugSize, +) +stats := mgr.GetInstanceStats(ctx, info) +``` + +## Prometheus Queries + +```promql +# CPU utilization rate (per vCPU) +rate(hypeman_vm_cpu_seconds_total[1m]) / hypeman_vm_allocated_vcpus + +# Memory utilization percentage +hypeman_vm_memory_rss_bytes / hypeman_vm_allocated_memory_bytes * 100 + +# Network throughput (bytes/sec) +rate(hypeman_vm_network_rx_bytes_total[1m]) +rate(hypeman_vm_network_tx_bytes_total[1m]) +``` diff --git a/lib/vm_metrics/adapter.go b/lib/vm_metrics/adapter.go new file mode 100644 index 00000000..0f18a187 --- /dev/null +++ b/lib/vm_metrics/adapter.go @@ -0,0 +1,92 @@ +package vm_metrics + +import ( + "context" + + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/resources" +) + +// InstanceManagerAdapter adapts an instance manager that returns resources.InstanceUtilizationInfo +// to the vm_metrics.InstanceSource interface. +type InstanceManagerAdapter struct { + manager interface { + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) + } +} + +// NewInstanceManagerAdapter creates an adapter for the given instance manager. +func NewInstanceManagerAdapter(manager interface { + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) +}) *InstanceManagerAdapter { + return &InstanceManagerAdapter{manager: manager} +} + +// ListRunningInstancesForMetrics implements InstanceSource. +func (a *InstanceManagerAdapter) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + ctx := context.Background() + infos, err := a.manager.ListRunningInstancesInfo(ctx) + if err != nil { + return nil, err + } + + result := make([]InstanceInfo, len(infos)) + for i, info := range infos { + result[i] = InstanceInfo{ + ID: info.ID, + Name: info.Name, + HypervisorPID: info.HypervisorPID, + TAPDevice: info.TAPDevice, + AllocatedVcpus: info.AllocatedVcpus, + AllocatedMemoryBytes: info.AllocatedMemoryBytes, + } + } + return result, nil +} + +// InstanceListerAdapter adapts an instance lister that provides Instance structs +// to the vm_metrics.InstanceSource interface. +// This is useful when you need to build InstanceInfo directly from Instance data. +type InstanceListerAdapter struct { + listFunc func(ctx context.Context) ([]InstanceData, error) +} + +// InstanceData contains the minimal instance data needed for metrics. +type InstanceData struct { + ID string + Name string + HypervisorPID *int + NetworkEnabled bool + AllocatedVcpus int + AllocatedMemoryBytes int64 +} + +// NewInstanceListerAdapter creates an adapter with a custom list function. +func NewInstanceListerAdapter(listFunc func(ctx context.Context) ([]InstanceData, error)) *InstanceListerAdapter { + return &InstanceListerAdapter{listFunc: listFunc} +} + +// ListRunningInstancesForMetrics implements InstanceSource. +func (a *InstanceListerAdapter) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + ctx := context.Background() + instances, err := a.listFunc(ctx) + if err != nil { + return nil, err + } + + result := make([]InstanceInfo, len(instances)) + for i, inst := range instances { + info := InstanceInfo{ + ID: inst.ID, + Name: inst.Name, + HypervisorPID: inst.HypervisorPID, + AllocatedVcpus: inst.AllocatedVcpus, + AllocatedMemoryBytes: inst.AllocatedMemoryBytes, + } + if inst.NetworkEnabled { + info.TAPDevice = network.GenerateTAPName(inst.ID) + } + result[i] = info + } + return result, nil +} diff --git a/lib/vm_metrics/collector.go b/lib/vm_metrics/collector.go new file mode 100644 index 00000000..f9ba89b9 --- /dev/null +++ b/lib/vm_metrics/collector.go @@ -0,0 +1,128 @@ +package vm_metrics + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// userHZ is the clock tick rate used by /proc for CPU times. +// This is USER_HZ (not kernel CONFIG_HZ), which is always 100 on Linux. +// The kernel converts internal HZ to USER_HZ when writing to /proc to +// maintain a stable userspace ABI. This has been 100 since Linux 2.4. +// See: https://man7.org/linux/man-pages/man5/proc.5.html (search for "clock ticks") +const userHZ = 100 + +// ReadProcStat reads CPU time from /proc//stat. +// Returns total CPU time (user + system) in microseconds. +// Fields 14 and 15 are utime and stime in clock ticks. +func ReadProcStat(pid int) (uint64, error) { + statPath := fmt.Sprintf("/proc/%d/stat", pid) + data, err := os.ReadFile(statPath) + if err != nil { + return 0, fmt.Errorf("read proc stat: %w", err) + } + + // /proc//stat format: pid (comm) state ppid ... field14 field15 ... + // We need to handle comm which may contain spaces and parentheses + content := string(data) + + // Find the last ')' to skip past the comm field + lastParen := strings.LastIndex(content, ")") + if lastParen == -1 { + return 0, fmt.Errorf("invalid proc stat format: no closing paren") + } + + // Fields after comm start at index 2 (0-indexed: state is field 2) + // utime is field 13 (0-indexed), stime is field 14 (0-indexed) + // After the ')', fields are space-separated starting from field 2 + fields := strings.Fields(content[lastParen+1:]) + if len(fields) < 13 { + return 0, fmt.Errorf("invalid proc stat format: not enough fields") + } + + // fields[11] = utime (field 14 in 1-indexed stat, but field 11 after comm) + // fields[12] = stime (field 15 in 1-indexed stat, but field 12 after comm) + utime, err := strconv.ParseUint(fields[11], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse utime: %w", err) + } + + stime, err := strconv.ParseUint(fields[12], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse stime: %w", err) + } + + // Convert clock ticks to microseconds + // /proc reports CPU times in USER_HZ (always 100 on Linux) + const usecPerTick = 1_000_000 / userHZ + + totalUsec := (utime + stime) * usecPerTick + return totalUsec, nil +} + +// ReadProcStatm reads memory stats from /proc//statm. +// Returns RSS (resident set size) and VMS (virtual memory size) in bytes. +// Format: size resident shared text lib data dt (all in pages) +func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { + statmPath := fmt.Sprintf("/proc/%d/statm", pid) + data, err := os.ReadFile(statmPath) + if err != nil { + return 0, 0, fmt.Errorf("read proc statm: %w", err) + } + + fields := strings.Fields(string(data)) + if len(fields) < 2 { + return 0, 0, fmt.Errorf("invalid proc statm format") + } + + // Field 0: size (total virtual memory in pages) + // Field 1: resident (resident set size in pages) + vmsPages, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse vms: %w", err) + } + + rssPages, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse rss: %w", err) + } + + // Convert pages to bytes using system page size (varies by architecture) + // x86_64: typically 4KB, ARM64: can be 4KB, 16KB, or 64KB + pageSize := uint64(os.Getpagesize()) + return rssPages * pageSize, vmsPages * pageSize, nil +} + +// ReadTAPStats reads network statistics from a TAP device. +// Reads from /sys/class/net//statistics/{rx,tx}_bytes. +// Note: Returns stats from host perspective. Caller must swap for VM perspective: +// - rxBytes = host receives = VM transmits +// - txBytes = host transmits = VM receives +func ReadTAPStats(tapName string) (rxBytes, txBytes uint64, err error) { + basePath := filepath.Join("/sys/class/net", tapName, "statistics") + + // Read RX bytes + rxData, err := os.ReadFile(filepath.Join(basePath, "rx_bytes")) + if err != nil { + return 0, 0, fmt.Errorf("read rx_bytes: %w", err) + } + rxBytes, err = strconv.ParseUint(strings.TrimSpace(string(rxData)), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse rx_bytes: %w", err) + } + + // Read TX bytes + txData, err := os.ReadFile(filepath.Join(basePath, "tx_bytes")) + if err != nil { + return 0, 0, fmt.Errorf("read tx_bytes: %w", err) + } + txBytes, err = strconv.ParseUint(strings.TrimSpace(string(txData)), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse tx_bytes: %w", err) + } + + return rxBytes, txBytes, nil +} diff --git a/lib/vm_metrics/collector_test.go b/lib/vm_metrics/collector_test.go new file mode 100644 index 00000000..6905a35b --- /dev/null +++ b/lib/vm_metrics/collector_test.go @@ -0,0 +1,72 @@ +package vm_metrics + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadProcStat(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + cpuUsec, err := ReadProcStat(pid) + require.NoError(t, err) + assert.True(t, cpuUsec >= 0, "CPU time should be non-negative") +} + +func TestReadProcStat_InvalidPID(t *testing.T) { + _, err := ReadProcStat(999999999) + assert.Error(t, err) +} + +func TestReadProcStatm(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + rssBytes, vmsBytes, err := ReadProcStatm(pid) + require.NoError(t, err) + assert.True(t, rssBytes > 0, "RSS should be positive") + assert.True(t, vmsBytes > 0, "VMS should be positive") + assert.True(t, vmsBytes >= rssBytes, "VMS should be >= RSS") +} + +func TestReadProcStatm_InvalidPID(t *testing.T) { + _, _, err := ReadProcStatm(999999999) + assert.Error(t, err) +} + +func TestReadProcStatm_PageSize(t *testing.T) { + // Verify that the page size is being used correctly + // The returned values should be multiples of the page size + pid := os.Getpid() + rssBytes, vmsBytes, err := ReadProcStatm(pid) + require.NoError(t, err) + + pageSize := uint64(os.Getpagesize()) + // RSS and VMS should be exact multiples of page size + assert.Equal(t, uint64(0), rssBytes%pageSize, "RSS should be a multiple of page size") + assert.Equal(t, uint64(0), vmsBytes%pageSize, "VMS should be a multiple of page size") +} + +func TestReadTAPStats(t *testing.T) { + // This test requires /sys/class/net to exist + // We'll use loopback which should always exist + testInterface := "lo" + + basePath := filepath.Join("/sys/class/net", testInterface, "statistics") + if _, err := os.Stat(basePath); os.IsNotExist(err) { + t.Skip("skipping test: /sys/class/net not available") + } + + rxBytes, txBytes, err := ReadTAPStats(testInterface) + require.NoError(t, err) + // Loopback should have some traffic (or at least zero is valid) + assert.True(t, rxBytes >= 0 || txBytes >= 0, "should be able to read stats") +} + +func TestReadTAPStats_NotExists(t *testing.T) { + _, _, err := ReadTAPStats("nonexistent-tap-device") + assert.Error(t, err) +} diff --git a/lib/vm_metrics/manager.go b/lib/vm_metrics/manager.go new file mode 100644 index 00000000..112047d8 --- /dev/null +++ b/lib/vm_metrics/manager.go @@ -0,0 +1,146 @@ +package vm_metrics + +import ( + "context" + "sync" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" + "go.opentelemetry.io/otel/metric" +) + +// Manager collects and exposes VM resource utilization metrics. +// It reads from /proc and TAP interfaces to gather real-time statistics. +type Manager struct { + mu sync.RWMutex + source InstanceSource + otel *otelMetrics +} + +// NewManager creates a new VM metrics manager. +func NewManager() *Manager { + return &Manager{} +} + +// SetInstanceSource sets the source for instance information. +// Must be called before collecting metrics. +func (m *Manager) SetInstanceSource(source InstanceSource) { + m.mu.Lock() + defer m.mu.Unlock() + m.source = source +} + +// InitializeOTel sets up OpenTelemetry metrics. +// If meter is nil, OTel metrics are disabled. +func (m *Manager) InitializeOTel(meter metric.Meter) error { + if meter == nil { + return nil + } + + otel, err := newOTelMetrics(meter, m) + if err != nil { + return err + } + + m.mu.Lock() + m.otel = otel + m.mu.Unlock() + + return nil +} + +// GetInstanceStats collects metrics for a single instance. +// Returns nil if the instance is not running or stats cannot be collected. +func (m *Manager) GetInstanceStats(ctx context.Context, info InstanceInfo) *VMStats { + log := logger.FromContext(ctx) + + stats := &VMStats{ + InstanceID: info.ID, + InstanceName: info.Name, + AllocatedVcpus: info.AllocatedVcpus, + AllocatedMemoryBytes: info.AllocatedMemoryBytes, + } + + // Read /proc stats if we have a hypervisor PID + if info.HypervisorPID != nil { + pid := *info.HypervisorPID + + // Read CPU from /proc//stat + cpuUsec, err := ReadProcStat(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc stat", "instance_id", info.ID, "pid", pid, "error", err) + } else { + stats.CPUUsec = cpuUsec + } + + // Read memory from /proc//statm + rssBytes, vmsBytes, err := ReadProcStatm(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc statm", "instance_id", info.ID, "pid", pid, "error", err) + } else { + stats.MemoryRSSBytes = rssBytes + stats.MemoryVMSBytes = vmsBytes + } + } + + // Read TAP stats if we have a TAP device + if info.TAPDevice != "" { + rxBytes, txBytes, err := ReadTAPStats(info.TAPDevice) + if err != nil { + log.DebugContext(ctx, "failed to read TAP stats", "instance_id", info.ID, "tap", info.TAPDevice, "error", err) + } else { + // TAP stats are from host perspective, swap for VM perspective: + // - TAP rx_bytes = host receives = VM transmits + // - TAP tx_bytes = host transmits = VM receives + stats.NetRxBytes = txBytes + stats.NetTxBytes = rxBytes + } + } + + return stats +} + +// CollectAll gathers metrics for all running VMs. +// Used by OTel metrics callback. +func (m *Manager) CollectAll(ctx context.Context) ([]VMStats, error) { + m.mu.RLock() + source := m.source + m.mu.RUnlock() + + if source == nil { + return nil, nil + } + + instances, err := source.ListRunningInstancesForMetrics() + if err != nil { + return nil, err + } + + var stats []VMStats + for _, info := range instances { + s := m.GetInstanceStats(ctx, info) + if s != nil { + stats = append(stats, *s) + } + } + + return stats, nil +} + +// BuildInstanceInfo creates an InstanceInfo from instance metadata. +// This is a helper for the API layer to avoid duplicating TAP name logic. +func BuildInstanceInfo(id, name string, pid *int, networkEnabled bool, vcpus int, memoryBytes int64) InstanceInfo { + info := InstanceInfo{ + ID: id, + Name: name, + HypervisorPID: pid, + AllocatedVcpus: vcpus, + AllocatedMemoryBytes: memoryBytes, + } + + if networkEnabled { + info.TAPDevice = network.GenerateTAPName(id) + } + + return info +} diff --git a/lib/vm_metrics/manager_test.go b/lib/vm_metrics/manager_test.go new file mode 100644 index 00000000..851771e7 --- /dev/null +++ b/lib/vm_metrics/manager_test.go @@ -0,0 +1,202 @@ +package vm_metrics + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockInstanceSource implements InstanceSource for testing +type mockInstanceSource struct { + instances []InstanceInfo +} + +func (m *mockInstanceSource) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + return m.instances, nil +} + +func TestManager_GetInstanceStats(t *testing.T) { + mgr := NewManager() + + // Test with no PID - should return stats with zero values + info := InstanceInfo{ + ID: "test-instance-1", + Name: "test-vm", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 2, + AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB + } + + stats := mgr.GetInstanceStats(context.Background(), info) + require.NotNil(t, stats) + assert.Equal(t, "test-instance-1", stats.InstanceID) + assert.Equal(t, "test-vm", stats.InstanceName) + assert.Equal(t, 2, stats.AllocatedVcpus) + assert.Equal(t, int64(1024*1024*1024), stats.AllocatedMemoryBytes) + assert.Equal(t, uint64(0), stats.CPUUsec) + assert.Equal(t, uint64(0), stats.MemoryRSSBytes) +} + +func TestManager_GetInstanceStats_WithCurrentProcess(t *testing.T) { + mgr := NewManager() + pid := os.Getpid() + + info := InstanceInfo{ + ID: "test-instance", + Name: "test-vm", + HypervisorPID: &pid, + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB + } + + stats := mgr.GetInstanceStats(context.Background(), info) + require.NotNil(t, stats) + + // Should have non-zero values since we're reading from current process + assert.True(t, stats.CPUUsec >= 0, "CPU time should be non-negative") + assert.True(t, stats.MemoryRSSBytes > 0, "RSS should be positive") + assert.True(t, stats.MemoryVMSBytes > 0, "VMS should be positive") +} + +func TestManager_CollectAll_NilSource(t *testing.T) { + mgr := NewManager() + + // Test with nil source - should return nil, no error + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + assert.Nil(t, stats) +} + +func TestManager_CollectAll_EmptySource(t *testing.T) { + mgr := NewManager() + mgr.SetInstanceSource(&mockInstanceSource{instances: []InstanceInfo{}}) + + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + assert.Empty(t, stats) +} + +func TestManager_CollectAll_MultipleInstances(t *testing.T) { + mgr := NewManager() + mgr.SetInstanceSource(&mockInstanceSource{ + instances: []InstanceInfo{ + { + ID: "vm-001", + Name: "web-server", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, + }, + { + ID: "vm-002", + Name: "database", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 8, + AllocatedMemoryBytes: 16 * 1024 * 1024 * 1024, + }, + }, + }) + + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + require.Len(t, stats, 2) + + assert.Equal(t, "vm-001", stats[0].InstanceID) + assert.Equal(t, "web-server", stats[0].InstanceName) + assert.Equal(t, 4, stats[0].AllocatedVcpus) + + assert.Equal(t, "vm-002", stats[1].InstanceID) + assert.Equal(t, "database", stats[1].InstanceName) + assert.Equal(t, 8, stats[1].AllocatedVcpus) +} + +func TestVMStats_CPUSeconds(t *testing.T) { + stats := &VMStats{ + CPUUsec: 1500000, // 1.5 seconds in microseconds + } + assert.InDelta(t, 1.5, stats.CPUSeconds(), 0.001) +} + +func TestVMStats_MemoryUtilizationRatio(t *testing.T) { + tests := []struct { + name string + rss uint64 + allocated int64 + expectRatio *float64 + expectNil bool + }{ + { + name: "normal ratio", + rss: 536870912, // 512MB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(0.5), + }, + { + name: "100% utilization", + rss: 1073741824, // 1GB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(1.0), + }, + { + name: "over 100% utilization", + rss: 2147483648, // 2GB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(2.0), + }, + { + name: "zero allocated", + rss: 536870912, + allocated: 0, + expectNil: true, + }, + { + name: "negative allocated", + rss: 536870912, + allocated: -1, + expectNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stats := &VMStats{ + MemoryRSSBytes: tt.rss, + AllocatedMemoryBytes: tt.allocated, + } + ratio := stats.MemoryUtilizationRatio() + if tt.expectNil { + assert.Nil(t, ratio) + } else { + require.NotNil(t, ratio) + assert.InDelta(t, *tt.expectRatio, *ratio, 0.001) + } + }) + } +} + +func TestBuildInstanceInfo(t *testing.T) { + pid := 1234 + + // With network enabled + info := BuildInstanceInfo("abc123", "my-vm", &pid, true, 4, 4*1024*1024*1024) + assert.Equal(t, "abc123", info.ID) + assert.Equal(t, "my-vm", info.Name) + assert.Equal(t, &pid, info.HypervisorPID) + assert.Equal(t, 4, info.AllocatedVcpus) + assert.Equal(t, int64(4*1024*1024*1024), info.AllocatedMemoryBytes) + assert.NotEmpty(t, info.TAPDevice, "should have TAP device when network enabled") + + // Without network enabled + info = BuildInstanceInfo("abc123", "my-vm", &pid, false, 4, 4*1024*1024*1024) + assert.Empty(t, info.TAPDevice, "should not have TAP device when network disabled") +} + +func ptrFloat64(v float64) *float64 { + return &v +} diff --git a/lib/vm_metrics/metrics.go b/lib/vm_metrics/metrics.go new file mode 100644 index 00000000..6bb64e90 --- /dev/null +++ b/lib/vm_metrics/metrics.go @@ -0,0 +1,163 @@ +package vm_metrics + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// otelMetrics holds the OpenTelemetry instruments for VM metrics. +type otelMetrics struct { + cpuSecondsTotal metric.Float64ObservableCounter + allocatedVcpus metric.Int64ObservableGauge + memoryRSSBytes metric.Int64ObservableGauge + memoryVMSBytes metric.Int64ObservableGauge + allocatedMemoryBytes metric.Int64ObservableGauge + networkRxBytesTotal metric.Int64ObservableCounter + networkTxBytesTotal metric.Int64ObservableCounter + memoryUtilizationRatio metric.Float64ObservableGauge +} + +// newOTelMetrics creates and registers all VM utilization metrics. +func newOTelMetrics(meter metric.Meter, m *Manager) (*otelMetrics, error) { + // CPU time in seconds (converted from microseconds) + cpuSecondsTotal, err := meter.Float64ObservableCounter( + "hypeman_vm_cpu_seconds_total", + metric.WithDescription("Total CPU time consumed by the VM hypervisor process in seconds"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + // Allocated vCPUs + allocatedVcpus, err := meter.Int64ObservableGauge( + "hypeman_vm_allocated_vcpus", + metric.WithDescription("Number of vCPUs allocated to the VM"), + metric.WithUnit("{vcpu}"), + ) + if err != nil { + return nil, err + } + + // Memory RSS (Resident Set Size) - actual physical memory used + memoryRSSBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_memory_rss_bytes", + metric.WithDescription("Resident Set Size - actual physical memory used by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Memory VMS (Virtual Memory Size) - total allocated virtual memory + memoryVMSBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_memory_vms_bytes", + metric.WithDescription("Virtual Memory Size - total virtual memory allocated for the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Allocated memory bytes + allocatedMemoryBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_allocated_memory_bytes", + metric.WithDescription("Total memory allocated to the VM (Size + HotplugSize)"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Network RX bytes (from TAP - bytes received by VM) + networkRxBytesTotal, err := meter.Int64ObservableCounter( + "hypeman_vm_network_rx_bytes_total", + metric.WithDescription("Total network bytes received by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Network TX bytes (from TAP - bytes transmitted by VM) + networkTxBytesTotal, err := meter.Int64ObservableCounter( + "hypeman_vm_network_tx_bytes_total", + metric.WithDescription("Total network bytes transmitted by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Memory utilization ratio (RSS / allocated) + memoryUtilizationRatio, err := meter.Float64ObservableGauge( + "hypeman_vm_memory_utilization_ratio", + metric.WithDescription("Memory utilization ratio (RSS / allocated memory)"), + metric.WithUnit("1"), + ) + if err != nil { + return nil, err + } + + // Register the callback that will collect all utilization metrics + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + stats, err := m.CollectAll(ctx) + if err != nil { + // Log error but don't fail the callback + return nil + } + + for _, s := range stats { + attrs := metric.WithAttributes( + attribute.String("instance_id", s.InstanceID), + attribute.String("instance_name", s.InstanceName), + ) + + // CPU time in seconds + o.ObserveFloat64(cpuSecondsTotal, s.CPUSeconds(), attrs) + + // Allocated resources + o.ObserveInt64(allocatedVcpus, int64(s.AllocatedVcpus), attrs) + o.ObserveInt64(allocatedMemoryBytes, s.AllocatedMemoryBytes, attrs) + + // Actual usage + o.ObserveInt64(memoryRSSBytes, int64(s.MemoryRSSBytes), attrs) + o.ObserveInt64(memoryVMSBytes, int64(s.MemoryVMSBytes), attrs) + o.ObserveInt64(networkRxBytesTotal, int64(s.NetRxBytes), attrs) + o.ObserveInt64(networkTxBytesTotal, int64(s.NetTxBytes), attrs) + + // Compute utilization ratio + if ratio := s.MemoryUtilizationRatio(); ratio != nil { + o.ObserveFloat64(memoryUtilizationRatio, *ratio, attrs) + } + } + + return nil + }, + cpuSecondsTotal, + allocatedVcpus, + memoryRSSBytes, + memoryVMSBytes, + allocatedMemoryBytes, + networkRxBytesTotal, + networkTxBytesTotal, + memoryUtilizationRatio, + ) + if err != nil { + return nil, err + } + + return &otelMetrics{ + cpuSecondsTotal: cpuSecondsTotal, + allocatedVcpus: allocatedVcpus, + memoryRSSBytes: memoryRSSBytes, + memoryVMSBytes: memoryVMSBytes, + allocatedMemoryBytes: allocatedMemoryBytes, + networkRxBytesTotal: networkRxBytesTotal, + networkTxBytesTotal: networkTxBytesTotal, + memoryUtilizationRatio: memoryUtilizationRatio, + }, nil +} diff --git a/lib/vm_metrics/metrics_test.go b/lib/vm_metrics/metrics_test.go new file mode 100644 index 00000000..65bee0d3 --- /dev/null +++ b/lib/vm_metrics/metrics_test.go @@ -0,0 +1,75 @@ +package vm_metrics + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestOTelMetrics_Registration(t *testing.T) { + // Create a test meter + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + meter := provider.Meter("test") + + // Create manager with mock source + mgr := NewManager() + pid := os.Getpid() + mgr.SetInstanceSource(&mockInstanceSource{ + instances: []InstanceInfo{ + { + ID: "test-vm", + Name: "Test VM", + HypervisorPID: &pid, + AllocatedVcpus: 2, + AllocatedMemoryBytes: 2 * 1024 * 1024 * 1024, + }, + }, + }) + + // Initialize OTel + err := mgr.InitializeOTel(meter) + require.NoError(t, err) + + // Collect metrics + var rm metricdata.ResourceMetrics + err = reader.Collect(t.Context(), &rm) + require.NoError(t, err) + + // Verify we have scope metrics + require.NotEmpty(t, rm.ScopeMetrics, "should have scope metrics") + + // Find our metrics + metricNames := make(map[string]bool) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames[m.Name] = true + } + } + + // Check expected metrics are present + expectedMetrics := []string{ + "hypeman_vm_cpu_seconds_total", + "hypeman_vm_allocated_vcpus", + "hypeman_vm_memory_rss_bytes", + "hypeman_vm_memory_vms_bytes", + "hypeman_vm_allocated_memory_bytes", + "hypeman_vm_network_rx_bytes_total", + "hypeman_vm_network_tx_bytes_total", + "hypeman_vm_memory_utilization_ratio", + } + + for _, expected := range expectedMetrics { + assert.True(t, metricNames[expected], "should have metric %s", expected) + } +} + +func TestOTelMetrics_NilMeter(t *testing.T) { + mgr := NewManager() + err := mgr.InitializeOTel(nil) + require.NoError(t, err, "nil meter should not error") +} diff --git a/lib/vm_metrics/types.go b/lib/vm_metrics/types.go new file mode 100644 index 00000000..1e4b197d --- /dev/null +++ b/lib/vm_metrics/types.go @@ -0,0 +1,61 @@ +// Package vm_metrics provides real-time resource utilization metrics for VMs. +// It collects CPU, memory, and network statistics from the host's perspective +// by reading /proc//stat, /proc//statm, and TAP interface statistics. +package vm_metrics + +// VMStats holds resource utilization metrics for a single VM. +// These are point-in-time values collected from the hypervisor process. +type VMStats struct { + InstanceID string + InstanceName string + + // CPU stats (from /proc//stat) + CPUUsec uint64 // Total CPU time in microseconds (user + system) + + // Memory stats (from /proc//statm) + MemoryRSSBytes uint64 // Resident Set Size - actual physical memory used + MemoryVMSBytes uint64 // Virtual Memory Size - total allocated virtual memory + + // Network stats (from TAP interface) + NetRxBytes uint64 // Total network bytes received + NetTxBytes uint64 // Total network bytes transmitted + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes +} + +// CPUSeconds returns CPU time in seconds (for API responses). +func (s *VMStats) CPUSeconds() float64 { + return float64(s.CPUUsec) / 1_000_000.0 +} + +// MemoryUtilizationRatio returns RSS / allocated memory (0.0 to 1.0+). +// Returns nil if allocated memory is 0. +func (s *VMStats) MemoryUtilizationRatio() *float64 { + if s.AllocatedMemoryBytes <= 0 { + return nil + } + ratio := float64(s.MemoryRSSBytes) / float64(s.AllocatedMemoryBytes) + return &ratio +} + +// InstanceInfo contains the minimal info needed to collect VM metrics. +// This is provided by the instances package. +type InstanceInfo struct { + ID string + Name string + HypervisorPID *int // PID of the hypervisor process (nil if not running) + TAPDevice string // Name of the TAP device (e.g., "hype-01234567") + + // Allocated resources + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) +} + +// InstanceSource provides access to running instance information. +// Implemented by instances.Manager. +type InstanceSource interface { + // ListRunningInstancesForMetrics returns info for all running instances. + ListRunningInstancesForMetrics() ([]InstanceInfo, error) +} diff --git a/openapi.yaml b/openapi.yaml index 1a6bc6ff..ee637a4a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -333,6 +333,60 @@ components: nullable: true example: "permission denied" + InstanceStats: + type: object + required: [instance_id, instance_name, cpu_seconds, memory_rss_bytes, memory_vms_bytes, network_rx_bytes, network_tx_bytes, allocated_vcpus, allocated_memory_bytes] + description: Real-time resource utilization statistics for a VM instance + properties: + instance_id: + type: string + description: Instance identifier + example: "qilviffnqzck2jrim1x6s2b1" + instance_name: + type: string + description: Instance name + example: "my-web-server" + cpu_seconds: + type: number + format: double + description: Total CPU time consumed by the VM hypervisor process in seconds + example: 29.94 + memory_rss_bytes: + type: integer + format: int64 + description: Resident Set Size - actual physical memory used by the VM in bytes + example: 443338752 + memory_vms_bytes: + type: integer + format: int64 + description: Virtual Memory Size - total virtual memory allocated in bytes + example: 4330745856 + network_rx_bytes: + type: integer + format: int64 + description: Total network bytes received by the VM (from TAP interface) + example: 12345678 + network_tx_bytes: + type: integer + format: int64 + description: Total network bytes transmitted by the VM (from TAP interface) + example: 87654321 + allocated_vcpus: + type: integer + description: Number of vCPUs allocated to the VM + example: 2 + allocated_memory_bytes: + type: integer + format: int64 + description: Total memory allocated to the VM (Size + HotplugSize) in bytes + example: 4294967296 + memory_utilization_ratio: + type: number + format: double + description: Memory utilization ratio (RSS / allocated memory). Only present when allocated_memory_bytes > 0. + nullable: true + example: 0.103 + CreateImageRequest: type: object required: [name] @@ -1577,6 +1631,43 @@ paths: schema: $ref: "#/components/schemas/Error" + /instances/{id}/stats: + get: + summary: Get instance resource utilization stats + description: | + Returns real-time resource utilization statistics for a running VM instance. + Metrics are collected from /proc//stat and /proc//statm for CPU and memory, + and from TAP interface statistics for network I/O. + operationId: getInstanceStats + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Instance ID or name + responses: + 200: + description: Instance utilization statistics + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceStats" + 404: + description: Instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /instances/{id}/volumes/{volumeId}: post: summary: Attach volume to instance