From 44ecfaeeeb2af9934f8d6b1d355ac67d9b208806 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:02:24 -0500 Subject: [PATCH 1/7] feat(metrics): add per-VM resource utilization metrics Add real-time VM resource utilization metrics using /proc//stat and /proc//statm for accurate per-process measurements (instead of cgroups which aggregate at the session level). New metrics exported via OpenTelemetry: - hypeman_vm_cpu_seconds_total: CPU time consumed by VM hypervisor - hypeman_vm_allocated_vcpus: Number of vCPUs allocated - hypeman_vm_memory_rss_bytes: Resident Set Size (actual physical memory) - hypeman_vm_memory_vms_bytes: Virtual Memory Size - hypeman_vm_allocated_memory_bytes: Total allocated memory - hypeman_vm_network_rx_bytes_total: Network bytes received (from TAP) - hypeman_vm_network_tx_bytes_total: Network bytes transmitted (from TAP) - hypeman_vm_memory_utilization_ratio: RSS / allocated memory Also adds REST endpoint GET /instances/{id}/stats for per-instance stats. --- cmd/api/api/stats.go | 135 +++++++++++++ cmd/api/main.go | 11 ++ dashboards/hypeman.json | 106 ++++++++++ lib/instances/manager.go | 51 +++++ lib/instances/manager_test.go | 36 ++++ lib/providers/providers.go | 9 + lib/resources/resource.go | 32 ++++ lib/resources/utilization.go | 224 ++++++++++++++++++++++ lib/resources/utilization_metrics.go | 166 ++++++++++++++++ lib/resources/utilization_metrics_test.go | 152 +++++++++++++++ lib/resources/utilization_test.go | 194 +++++++++++++++++++ 11 files changed, 1116 insertions(+) create mode 100644 cmd/api/api/stats.go create mode 100644 lib/resources/utilization.go create mode 100644 lib/resources/utilization_metrics.go create mode 100644 lib/resources/utilization_metrics_test.go create mode 100644 lib/resources/utilization_test.go diff --git a/cmd/api/api/stats.go b/cmd/api/api/stats.go new file mode 100644 index 00000000..0675e42e --- /dev/null +++ b/cmd/api/api/stats.go @@ -0,0 +1,135 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/logger" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/resources" +) + +// InstanceStats represents utilization statistics for a single instance +type InstanceStats struct { + InstanceID string `json:"instance_id"` + InstanceName string `json:"instance_name"` + + // CPU stats + CPUSeconds float64 `json:"cpu_seconds"` // Total CPU time consumed + + // Memory stats (from /proc//statm) + MemoryRSSBytes uint64 `json:"memory_rss_bytes"` // Resident Set Size - actual physical memory + MemoryVMSBytes uint64 `json:"memory_vms_bytes"` // Virtual Memory Size - total virtual memory + + // Network stats (from TAP interface) + NetworkRxBytes uint64 `json:"network_rx_bytes"` // Total bytes received + NetworkTxBytes uint64 `json:"network_tx_bytes"` // Total bytes transmitted + + // Allocated resources + AllocatedVcpus int `json:"allocated_vcpus"` + AllocatedMemoryBytes int64 `json:"allocated_memory_bytes"` + + // Utilization ratios + MemoryUtilizationRatio *float64 `json:"memory_utilization_ratio,omitempty"` // RSS / allocated +} + +// StatsHandler handles GET /instances/{id}/stats requests +func (s *ApiService) StatsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := logger.FromContext(ctx) + + // Get resolved instance from context (set by ResolveResource middleware) + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + http.Error(w, `{"error": "instance not found"}`, http.StatusNotFound) + return + } + + // Build InstanceUtilizationInfo for this specific instance + info := resources.InstanceUtilizationInfo{ + ID: inst.Id, + Name: inst.Name, + } + + // Get hypervisor PID if running + if inst.HypervisorPID != nil { + info.HypervisorPID = inst.HypervisorPID + } + + // Get allocated resources + info.AllocatedVcpus = inst.Vcpus + + // Calculate allocated memory (Size + HotplugSize) + info.AllocatedMemoryBytes = inst.Size + inst.HotplugSize + + // Get TAP device if network enabled + if inst.NetworkEnabled { + info.TAPDevice = generateTAPName(inst.Id) + } + + // Collect stats directly for this instance + stats := InstanceStats{ + InstanceID: inst.Id, + InstanceName: inst.Name, + AllocatedVcpus: info.AllocatedVcpus, + AllocatedMemoryBytes: info.AllocatedMemoryBytes, + } + + // Read /proc stats if we have a PID + if info.HypervisorPID != nil { + pid := *info.HypervisorPID + + // Read CPU from /proc//stat + cpuUsec, err := resources.ReadProcStat(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc stat", "pid", pid, "error", err) + } else { + stats.CPUSeconds = float64(cpuUsec) / 1_000_000.0 + } + + // Read memory from /proc//statm + rssBytes, vmsBytes, err := resources.ReadProcStatm(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc statm", "pid", pid, "error", err) + } else { + stats.MemoryRSSBytes = rssBytes + stats.MemoryVMSBytes = vmsBytes + + // Compute utilization ratio + if info.AllocatedMemoryBytes > 0 { + ratio := float64(rssBytes) / float64(info.AllocatedMemoryBytes) + stats.MemoryUtilizationRatio = &ratio + } + } + } + + // Read TAP stats if we have a TAP device + if info.TAPDevice != "" { + rxBytes, txBytes, err := resources.ReadTAPStats(info.TAPDevice) + if err != nil { + log.DebugContext(ctx, "failed to read TAP stats", "tap", info.TAPDevice, "error", err) + } else { + stats.NetworkRxBytes = rxBytes + stats.NetworkTxBytes = txBytes + } + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(stats); err != nil { + log.ErrorContext(ctx, "failed to encode stats response", "error", err) + } +} + +// generateTAPName generates TAP device name from instance ID (same logic as instances package) +func generateTAPName(instanceID string) string { + // TAP name format: "hype-" + first 8 chars of instance ID + // Max TAP name length is 15 chars (IFNAMSIZ - 1) + prefix := "hype-" + maxIDLen := 15 - len(prefix) // 10 chars available for ID + idPart := instanceID + if len(idPart) > maxIDLen { + idPart = idPart[:maxIDLen] + } + return prefix + idPart +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 39587e91..e336cf09 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -282,6 +282,17 @@ func run() error { mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), ).Get("/instances/{id}/cp", app.ApiService.CpHandler) + // Instance stats endpoint (outside OpenAPI spec) + r.With( + middleware.RequestID, + middleware.RealIP, + middleware.Recoverer, + mw.InjectLogger(logger), + mw.AccessLogger(accessLogger), + mw.JwtAuth(app.Config.JwtSecret), + mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), + ).Get("/instances/{id}/stats", app.ApiService.StatsHandler) + // OCI Distribution registry endpoints for image push (outside OpenAPI spec) r.Route("/v2", func(r chi.Router) { r.Use(middleware.RequestID) diff --git a/dashboards/hypeman.json b/dashboards/hypeman.json index c2c46c55..6e09676e 100644 --- a/dashboards/hypeman.json +++ b/dashboards/hypeman.json @@ -417,6 +417,112 @@ ], "title": "Exec Sessions & Duration", "type": "timeseries" + }, + { + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, + "id": 18, + "title": "VM Resource Utilization", + "type": "row" + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "id": 19, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_cpu_seconds_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}}", + "refId": "A" + } + ], + "title": "VM CPU Usage (cores)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "short", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "id": 20, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_rss_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}} RSS", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_vms_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}} VMS", + "refId": "B" + } + ], + "title": "VM Memory Usage (RSS & VMS)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "bytes", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, + "id": 21, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_network_rx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}} RX", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(hypeman_vm_network_tx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])", + "legendFormat": "{{instance_name}} TX", + "refId": "B" + } + ], + "title": "VM Network I/O", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "Bps", "min": 0 }, + "overrides": [] + } + }, + { + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }, + "id": 23, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "hypeman_vm_memory_utilization_ratio{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}", + "legendFormat": "{{instance_name}}", + "refId": "A" + } + ], + "title": "VM Memory Utilization (% of allocated)", + "type": "timeseries", + "fieldConfig": { + "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, + "overrides": [] + } } ], "refresh": "10s", diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 84daaa1e..35c42c05 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -3,6 +3,7 @@ package instances import ( "context" "fmt" + "strings" "sync" "github.com/kernel/hypeman/lib/devices" @@ -38,6 +39,9 @@ type Manager interface { // ListInstanceAllocations returns resource allocations for all instances. // Used by the resource manager for capacity tracking. ListInstanceAllocations(ctx context.Context) ([]resources.InstanceAllocation, error) + // ListRunningInstancesInfo returns info needed for utilization metrics collection. + // Used by the resource manager for VM utilization tracking. + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) } // ResourceLimits contains configurable resource limits for instances @@ -328,3 +332,50 @@ func (m *manager) ListInstanceAllocations(ctx context.Context) ([]resources.Inst return allocations, nil } + +// ListRunningInstancesInfo returns info needed for utilization metrics collection. +// Used by the resource manager for VM utilization tracking. +func (m *manager) ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) { + instances, err := m.listInstances(ctx) + if err != nil { + return nil, err + } + + infos := make([]resources.InstanceUtilizationInfo, 0, len(instances)) + for _, inst := range instances { + // Only include running instances (they have a hypervisor process) + if inst.State != StateRunning { + continue + } + + info := resources.InstanceUtilizationInfo{ + ID: inst.Id, + Name: inst.Name, + HypervisorPID: inst.HypervisorPID, + // Include allocated resources for utilization ratio calculations + AllocatedVcpus: inst.Vcpus, + AllocatedMemoryBytes: inst.Size + inst.HotplugSize, + } + + // Derive TAP device name if networking is enabled + if inst.NetworkEnabled { + info.TAPDevice = generateTAPName(inst.Id) + } + + infos = append(infos, info) + } + + return infos, nil +} + +// generateTAPName generates TAP device name from instance ID. +// This matches the logic in network/allocate.go. +func generateTAPName(instanceID string) string { + // Use first 8 chars of instance ID + // hype-{8chars} fits within 15-char Linux interface name limit + shortID := instanceID + if len(shortID) > 8 { + shortID = shortID[:8] + } + return "hype-" + strings.ToLower(shortID) +} diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 67037b33..1ee8ffee 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -1001,3 +1001,39 @@ func (r *testInstanceResolver) ResolveInstance(ctx context.Context, nameOrID str // For tests, just return nameOrID as both name and id return nameOrID, nameOrID, nil } + +func TestGenerateTAPName(t *testing.T) { + tests := []struct { + name string + instanceID string + expected string + }{ + { + name: "standard ID", + instanceID: "01HQVX7ABC123DEF456", + expected: "hype-01hqvx7a", + }, + { + name: "short ID", + instanceID: "ABC123", + expected: "hype-abc123", + }, + { + name: "exact 8 chars", + instanceID: "ABCDEFGH", + expected: "hype-abcdefgh", + }, + { + name: "mixed case", + instanceID: "AbCdEfGhIjKl", + expected: "hype-abcdefgh", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := generateTAPName(tt.instanceID) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 9f0bcd06..dc6c330e 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -157,11 +157,20 @@ func ProvideResourceManager(ctx context.Context, cfg *config.Config, p *paths.Pa mgr.SetInstanceLister(instanceManager) mgr.SetVolumeLister(volumeManager) + // Set utilization source for VM metrics (instanceManager implements UtilizationSource) + mgr.SetUtilizationSource(instanceManager) + // Initialize resource discovery if err := mgr.Initialize(ctx); err != nil { return nil, fmt.Errorf("initialize resource manager: %w", err) } + // Initialize VM utilization metrics + meter := otel.GetMeterProvider().Meter("hypeman") + if err := mgr.InitializeMetrics(meter); err != nil { + return nil, fmt.Errorf("initialize utilization metrics: %w", err) + } + return mgr, nil } diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 3bd9ee2d..c60c6e39 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -10,6 +10,7 @@ import ( "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/paths" + "go.opentelemetry.io/otel/metric" ) // ResourceType identifies a type of host resource. @@ -129,6 +130,10 @@ type Manager struct { instanceLister InstanceLister imageLister ImageLister volumeLister VolumeLister + + // Dependencies for utilization metrics + utilizationSource UtilizationSource + utilizationMetrics *UtilizationMetrics } // NewManager creates a new resource manager. @@ -161,6 +166,33 @@ func (m *Manager) SetVolumeLister(lister VolumeLister) { m.volumeLister = lister } +// SetUtilizationSource sets the utilization source for VM metrics collection. +func (m *Manager) SetUtilizationSource(source UtilizationSource) { + m.mu.Lock() + defer m.mu.Unlock() + m.utilizationSource = source +} + +// InitializeMetrics sets up OTel metrics for VM utilization. +// Must be called after SetUtilizationSource and before the manager is used. +// If meter is nil, metrics are not initialized. +func (m *Manager) InitializeMetrics(meter metric.Meter) error { + if meter == nil { + return nil + } + + metrics, err := newUtilizationMetrics(meter, m) + if err != nil { + return fmt.Errorf("initialize utilization metrics: %w", err) + } + + m.mu.Lock() + m.utilizationMetrics = metrics + m.mu.Unlock() + + return nil +} + // Initialize discovers host resources and registers them. // Must be called after setting listers and before using the manager. func (m *Manager) Initialize(ctx context.Context) error { diff --git a/lib/resources/utilization.go b/lib/resources/utilization.go new file mode 100644 index 00000000..15751d5d --- /dev/null +++ b/lib/resources/utilization.go @@ -0,0 +1,224 @@ +// Package resources provides host resource discovery, capacity tracking, +// and oversubscription-aware allocation management for CPU, memory, disk, and network. +package resources + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/kernel/hypeman/lib/logger" +) + +// VMUtilization holds actual resource utilization metrics for a VM. +// These are real-time values read from /proc//stat, /proc//statm, and TAP interfaces. +type VMUtilization struct { + InstanceID string + InstanceName string + CPUUsec uint64 // Total CPU time in microseconds (user + system) + MemoryRSSBytes uint64 // Resident Set Size - actual physical memory used + MemoryVMSBytes uint64 // Virtual Memory Size - total allocated virtual memory + NetRxBytes uint64 // Total network bytes received + NetTxBytes uint64 // Total network bytes transmitted + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes +} + +// UtilizationSource provides access to instance data for utilization collection. +type UtilizationSource interface { + // ListRunningInstancesInfo returns basic info for running instances. + ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) +} + +// InstanceUtilizationInfo contains the minimal info needed to collect utilization. +type InstanceUtilizationInfo struct { + ID string + Name string + HypervisorPID *int // PID of the hypervisor process + TAPDevice string // Name of the TAP device (e.g., "hype-01234567") + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) +} + +// CollectVMUtilization gathers utilization metrics for all running VMs. +// Uses /proc//stat and /proc//statm for per-process metrics (no cgroups needed). +func (m *Manager) CollectVMUtilization(ctx context.Context) ([]VMUtilization, error) { + m.mu.RLock() + source := m.utilizationSource + m.mu.RUnlock() + + if source == nil { + return nil, nil + } + + log := logger.FromContext(ctx) + + instances, err := source.ListRunningInstancesInfo(ctx) + if err != nil { + return nil, fmt.Errorf("list running instances: %w", err) + } + + var utilizations []VMUtilization + for _, inst := range instances { + util := VMUtilization{ + InstanceID: inst.ID, + InstanceName: inst.Name, + AllocatedVcpus: inst.AllocatedVcpus, + AllocatedMemoryBytes: inst.AllocatedMemoryBytes, + } + + // Collect per-process metrics from /proc if we have a PID + if inst.HypervisorPID != nil { + pid := *inst.HypervisorPID + + // Read CPU time from /proc//stat + cpuUsec, err := ReadProcStat(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc stat", "instance_id", inst.ID, "pid", pid, "error", err) + } else { + util.CPUUsec = cpuUsec + } + + // Read memory from /proc//statm + rssBytes, vmsBytes, err := ReadProcStatm(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc statm", "instance_id", inst.ID, "pid", pid, "error", err) + } else { + util.MemoryRSSBytes = rssBytes + util.MemoryVMSBytes = vmsBytes + } + } + + // Collect TAP stats if we have a TAP device + if inst.TAPDevice != "" { + rxBytes, txBytes, err := ReadTAPStats(inst.TAPDevice) + if err != nil { + log.DebugContext(ctx, "failed to read TAP stats", "instance_id", inst.ID, "tap", inst.TAPDevice, "error", err) + } else { + util.NetRxBytes = rxBytes + util.NetTxBytes = txBytes + } + } + + utilizations = append(utilizations, util) + } + + return utilizations, nil +} + +// ReadProcStat reads CPU time from /proc//stat. +// Returns total CPU time (user + system) in microseconds. +// Fields 14 and 15 are utime and stime in clock ticks. +func ReadProcStat(pid int) (uint64, error) { + statPath := fmt.Sprintf("/proc/%d/stat", pid) + data, err := os.ReadFile(statPath) + if err != nil { + return 0, fmt.Errorf("read proc stat: %w", err) + } + + // /proc//stat format: pid (comm) state ppid ... field14 field15 ... + // We need to handle comm which may contain spaces and parentheses + content := string(data) + + // Find the last ')' to skip past the comm field + lastParen := strings.LastIndex(content, ")") + if lastParen == -1 { + return 0, fmt.Errorf("invalid proc stat format: no closing paren") + } + + // Fields after comm start at index 2 (0-indexed: state is field 2) + // utime is field 13 (0-indexed), stime is field 14 (0-indexed) + // After the ')', fields are space-separated starting from field 2 + fields := strings.Fields(content[lastParen+1:]) + if len(fields) < 13 { + return 0, fmt.Errorf("invalid proc stat format: not enough fields") + } + + // fields[11] = utime (field 14 in 1-indexed stat, but field 11 after comm) + // fields[12] = stime (field 15 in 1-indexed stat, but field 12 after comm) + utime, err := strconv.ParseUint(fields[11], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse utime: %w", err) + } + + stime, err := strconv.ParseUint(fields[12], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse stime: %w", err) + } + + // Convert clock ticks to microseconds + // Clock ticks are typically 100 per second (sysconf(_SC_CLK_TCK)) + // 1 tick = 10000 microseconds (for 100 Hz) + const ticksPerSecond = 100 + const usecPerTick = 1_000_000 / ticksPerSecond + + totalUsec := (utime + stime) * usecPerTick + return totalUsec, nil +} + +// ReadProcStatm reads memory stats from /proc//statm. +// Returns RSS (resident set size) and VMS (virtual memory size) in bytes. +// Format: size resident shared text lib data dt (all in pages) +func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { + statmPath := fmt.Sprintf("/proc/%d/statm", pid) + data, err := os.ReadFile(statmPath) + if err != nil { + return 0, 0, fmt.Errorf("read proc statm: %w", err) + } + + fields := strings.Fields(string(data)) + if len(fields) < 2 { + return 0, 0, fmt.Errorf("invalid proc statm format") + } + + // Field 0: size (total virtual memory in pages) + // Field 1: resident (resident set size in pages) + vmsPages, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse vms: %w", err) + } + + rssPages, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse rss: %w", err) + } + + // Convert pages to bytes (page size is typically 4096) + const pageSize = 4096 + return rssPages * pageSize, vmsPages * pageSize, nil +} + +// ReadTAPStats reads network statistics from a TAP device. +// Reads from /sys/class/net//statistics/{rx,tx}_bytes. +func ReadTAPStats(tapName string) (rxBytes, txBytes uint64, err error) { + basePath := filepath.Join("/sys/class/net", tapName, "statistics") + + // Read RX bytes + rxData, err := os.ReadFile(filepath.Join(basePath, "rx_bytes")) + if err != nil { + return 0, 0, fmt.Errorf("read rx_bytes: %w", err) + } + rxBytes, err = strconv.ParseUint(strings.TrimSpace(string(rxData)), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse rx_bytes: %w", err) + } + + // Read TX bytes + txData, err := os.ReadFile(filepath.Join(basePath, "tx_bytes")) + if err != nil { + return 0, 0, fmt.Errorf("read tx_bytes: %w", err) + } + txBytes, err = strconv.ParseUint(strings.TrimSpace(string(txData)), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse tx_bytes: %w", err) + } + + return rxBytes, txBytes, nil +} diff --git a/lib/resources/utilization_metrics.go b/lib/resources/utilization_metrics.go new file mode 100644 index 00000000..274dba3d --- /dev/null +++ b/lib/resources/utilization_metrics.go @@ -0,0 +1,166 @@ +package resources + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// UtilizationMetrics holds the observable instruments for VM utilization. +type UtilizationMetrics struct { + cpuSecondsTotal metric.Float64ObservableCounter + allocatedVcpus metric.Int64ObservableGauge + memoryRSSBytes metric.Int64ObservableGauge + memoryVMSBytes metric.Int64ObservableGauge + allocatedMemoryBytes metric.Int64ObservableGauge + networkRxBytesTotal metric.Int64ObservableCounter + networkTxBytesTotal metric.Int64ObservableCounter + memoryUtilizationRatio metric.Float64ObservableGauge +} + +// newUtilizationMetrics creates and registers all VM utilization metrics. +// These are observable gauges/counters that read from /proc and TAP interfaces. +func newUtilizationMetrics(meter metric.Meter, m *Manager) (*UtilizationMetrics, error) { + // CPU time in seconds (converted from microseconds) + cpuSecondsTotal, err := meter.Float64ObservableCounter( + "hypeman_vm_cpu_seconds_total", + metric.WithDescription("Total CPU time consumed by the VM hypervisor process in seconds"), + metric.WithUnit("s"), + ) + if err != nil { + return nil, err + } + + // Allocated vCPUs + allocatedVcpus, err := meter.Int64ObservableGauge( + "hypeman_vm_allocated_vcpus", + metric.WithDescription("Number of vCPUs allocated to the VM"), + metric.WithUnit("{vcpu}"), + ) + if err != nil { + return nil, err + } + + // Memory RSS (Resident Set Size) - actual physical memory used + memoryRSSBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_memory_rss_bytes", + metric.WithDescription("Resident Set Size - actual physical memory used by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Memory VMS (Virtual Memory Size) - total allocated virtual memory + memoryVMSBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_memory_vms_bytes", + metric.WithDescription("Virtual Memory Size - total virtual memory allocated for the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Allocated memory bytes + allocatedMemoryBytes, err := meter.Int64ObservableGauge( + "hypeman_vm_allocated_memory_bytes", + metric.WithDescription("Total memory allocated to the VM (Size + HotplugSize)"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Network RX bytes (from TAP - bytes received by VM) + networkRxBytesTotal, err := meter.Int64ObservableCounter( + "hypeman_vm_network_rx_bytes_total", + metric.WithDescription("Total network bytes received by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Network TX bytes (from TAP - bytes transmitted by VM) + networkTxBytesTotal, err := meter.Int64ObservableCounter( + "hypeman_vm_network_tx_bytes_total", + metric.WithDescription("Total network bytes transmitted by the VM"), + metric.WithUnit("By"), + ) + if err != nil { + return nil, err + } + + // Memory utilization ratio (RSS / allocated) + memoryUtilizationRatio, err := meter.Float64ObservableGauge( + "hypeman_vm_memory_utilization_ratio", + metric.WithDescription("Memory utilization ratio (RSS / allocated memory)"), + metric.WithUnit("1"), + ) + if err != nil { + return nil, err + } + + // Register the callback that will collect all utilization metrics + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + utilizations, err := m.CollectVMUtilization(ctx) + if err != nil { + // Log error but don't fail the callback + return nil + } + + for _, util := range utilizations { + attrs := metric.WithAttributes( + attribute.String("instance_id", util.InstanceID), + attribute.String("instance_name", util.InstanceName), + ) + + // Convert CPU microseconds to seconds + cpuSeconds := float64(util.CPUUsec) / 1_000_000.0 + o.ObserveFloat64(cpuSecondsTotal, cpuSeconds, attrs) + + // Allocated resources + o.ObserveInt64(allocatedVcpus, int64(util.AllocatedVcpus), attrs) + o.ObserveInt64(allocatedMemoryBytes, util.AllocatedMemoryBytes, attrs) + + // Actual usage + o.ObserveInt64(memoryRSSBytes, int64(util.MemoryRSSBytes), attrs) + o.ObserveInt64(memoryVMSBytes, int64(util.MemoryVMSBytes), attrs) + o.ObserveInt64(networkRxBytesTotal, int64(util.NetRxBytes), attrs) + o.ObserveInt64(networkTxBytesTotal, int64(util.NetTxBytes), attrs) + + // Compute utilization ratio (RSS vs allocated) + if util.AllocatedMemoryBytes > 0 { + memRatio := float64(util.MemoryRSSBytes) / float64(util.AllocatedMemoryBytes) + o.ObserveFloat64(memoryUtilizationRatio, memRatio, attrs) + } + } + + return nil + }, + cpuSecondsTotal, + allocatedVcpus, + memoryRSSBytes, + memoryVMSBytes, + allocatedMemoryBytes, + networkRxBytesTotal, + networkTxBytesTotal, + memoryUtilizationRatio, + ) + if err != nil { + return nil, err + } + + return &UtilizationMetrics{ + cpuSecondsTotal: cpuSecondsTotal, + allocatedVcpus: allocatedVcpus, + memoryRSSBytes: memoryRSSBytes, + memoryVMSBytes: memoryVMSBytes, + allocatedMemoryBytes: allocatedMemoryBytes, + networkRxBytesTotal: networkRxBytesTotal, + networkTxBytesTotal: networkTxBytesTotal, + memoryUtilizationRatio: memoryUtilizationRatio, + }, nil +} diff --git a/lib/resources/utilization_metrics_test.go b/lib/resources/utilization_metrics_test.go new file mode 100644 index 00000000..1988bf31 --- /dev/null +++ b/lib/resources/utilization_metrics_test.go @@ -0,0 +1,152 @@ +package resources + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// mockUtilizationSourceWithData returns instances with simulated cgroup data +type mockUtilizationSourceWithData struct { + utilizations []VMUtilization +} + +func (m *mockUtilizationSourceWithData) ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) { + // Return info that will be used by CollectVMUtilization + // Since we can't easily mock the cgroup files in this test, we'll use a different approach + infos := make([]InstanceUtilizationInfo, len(m.utilizations)) + for i, u := range m.utilizations { + infos[i] = InstanceUtilizationInfo{ + ID: u.InstanceID, + Name: u.InstanceName, + AllocatedVcpus: u.AllocatedVcpus, + AllocatedMemoryBytes: u.AllocatedMemoryBytes, + } + } + return infos, nil +} + +func TestUtilizationMetrics_OTelIntegration(t *testing.T) { + // Create an in-memory metric reader + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + meter := provider.Meter("test") + + // Create manager + manager := &Manager{} + + // Initialize metrics + err := manager.InitializeMetrics(meter) + require.NoError(t, err) + + // Set up mock source + mockSource := &mockUtilizationSource{ + instances: []InstanceUtilizationInfo{ + { + ID: "test-vm-1", + Name: "web-app", + AllocatedVcpus: 2, + AllocatedMemoryBytes: 2 * 1024 * 1024 * 1024, + }, + }, + } + manager.SetUtilizationSource(mockSource) + + // Collect metrics + var rm metricdata.ResourceMetrics + err = reader.Collect(context.Background(), &rm) + require.NoError(t, err) + + // Verify we have scope metrics + require.NotEmpty(t, rm.ScopeMetrics, "should have scope metrics") + + // Find our metrics + var foundCPU, foundAllocVcpus, foundMemoryRSS, foundMemoryVMS, foundAllocMem, foundMemRatio bool + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + switch m.Name { + case "hypeman_vm_cpu_seconds_total": + foundCPU = true + case "hypeman_vm_allocated_vcpus": + foundAllocVcpus = true + case "hypeman_vm_memory_rss_bytes": + foundMemoryRSS = true + case "hypeman_vm_memory_vms_bytes": + foundMemoryVMS = true + case "hypeman_vm_allocated_memory_bytes": + foundAllocMem = true + case "hypeman_vm_memory_utilization_ratio": + foundMemRatio = true + } + } + } + + assert.True(t, foundCPU, "should have CPU metric registered") + assert.True(t, foundAllocVcpus, "should have allocated vCPUs metric registered") + assert.True(t, foundMemoryRSS, "should have memory RSS metric registered") + assert.True(t, foundMemoryVMS, "should have memory VMS metric registered") + assert.True(t, foundAllocMem, "should have allocated memory metric registered") + assert.True(t, foundMemRatio, "should have memory utilization ratio metric registered") +} + +func TestUtilizationMetrics_NilMeter(t *testing.T) { + manager := &Manager{} + + // Should not error with nil meter + err := manager.InitializeMetrics(nil) + require.NoError(t, err) +} + +func TestUtilizationMetrics_MetricNames(t *testing.T) { + // Verify all expected metric names are correct when data is present + expectedMetrics := []string{ + "hypeman_vm_cpu_seconds_total", + "hypeman_vm_allocated_vcpus", + "hypeman_vm_memory_rss_bytes", + "hypeman_vm_memory_vms_bytes", + "hypeman_vm_allocated_memory_bytes", + "hypeman_vm_network_rx_bytes_total", + "hypeman_vm_network_tx_bytes_total", + "hypeman_vm_memory_utilization_ratio", + } + + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + meter := provider.Meter("test") + + manager := &Manager{} + // Use mock source with actual data so metrics will be emitted + manager.SetUtilizationSource(&mockUtilizationSource{ + instances: []InstanceUtilizationInfo{ + { + ID: "test-vm", + Name: "test", + AllocatedVcpus: 2, + AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB + }, + }, + }) + err := manager.InitializeMetrics(meter) + require.NoError(t, err) + + var rm metricdata.ResourceMetrics + err = reader.Collect(context.Background(), &rm) + require.NoError(t, err) + + // Collect all metric names + var metricNames []string + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames = append(metricNames, m.Name) + } + } + + // Check all expected metrics are present + for _, expected := range expectedMetrics { + assert.Contains(t, metricNames, expected, "should have metric %s", expected) + } +} diff --git a/lib/resources/utilization_test.go b/lib/resources/utilization_test.go new file mode 100644 index 00000000..9435eb25 --- /dev/null +++ b/lib/resources/utilization_test.go @@ -0,0 +1,194 @@ +package resources + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadProcStat(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + cpuUsec, err := ReadProcStat(pid) + require.NoError(t, err) + assert.True(t, cpuUsec >= 0, "CPU time should be non-negative") +} + +func TestReadProcStat_InvalidPID(t *testing.T) { + _, err := ReadProcStat(999999999) + assert.Error(t, err) +} + +func TestReadProcStatm(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + rssBytes, vmsBytes, err := ReadProcStatm(pid) + require.NoError(t, err) + assert.True(t, rssBytes > 0, "RSS should be positive") + assert.True(t, vmsBytes > 0, "VMS should be positive") + assert.True(t, vmsBytes >= rssBytes, "VMS should be >= RSS") +} + +func TestReadProcStatm_InvalidPID(t *testing.T) { + _, _, err := ReadProcStatm(999999999) + assert.Error(t, err) +} + +func TestReadTAPStats(t *testing.T) { + // This test requires /sys/class/net to exist + // We'll use loopback which should always exist + testInterface := "lo" + + basePath := filepath.Join("/sys/class/net", testInterface, "statistics") + if _, err := os.Stat(basePath); os.IsNotExist(err) { + t.Skip("skipping test: /sys/class/net not available") + } + + rxBytes, txBytes, err := ReadTAPStats(testInterface) + require.NoError(t, err) + // Loopback should have some traffic (or at least zero is valid) + assert.True(t, rxBytes >= 0 || txBytes >= 0, "should be able to read stats") +} + +func TestReadTAPStats_NotExists(t *testing.T) { + _, _, err := ReadTAPStats("nonexistent-tap-device") + assert.Error(t, err) +} + +// mockUtilizationSource implements UtilizationSource for testing +type mockUtilizationSource struct { + instances []InstanceUtilizationInfo +} + +func (m *mockUtilizationSource) ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) { + return m.instances, nil +} + +func TestCollectVMUtilization_WithMockSource(t *testing.T) { + // Create a manager with mock source + manager := &Manager{} + + // Test with nil source - should return nil, no error + utils, err := manager.CollectVMUtilization(context.Background()) + require.NoError(t, err) + assert.Nil(t, utils) + + // Set up mock source with no running instances + mockSource := &mockUtilizationSource{ + instances: []InstanceUtilizationInfo{}, + } + manager.SetUtilizationSource(mockSource) + + utils, err = manager.CollectVMUtilization(context.Background()) + require.NoError(t, err) + assert.Empty(t, utils) + + // Test with instances that have no PID (simulates instances where proc reading will fail gracefully) + mockSource.instances = []InstanceUtilizationInfo{ + { + ID: "test-instance-1", + Name: "test-vm", + HypervisorPID: nil, // No PID - proc reading skipped + TAPDevice: "", // No TAP - network reading skipped + AllocatedVcpus: 2, + AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB + }, + } + + utils, err = manager.CollectVMUtilization(context.Background()) + require.NoError(t, err) + require.Len(t, utils, 1) + assert.Equal(t, "test-instance-1", utils[0].InstanceID) + assert.Equal(t, "test-vm", utils[0].InstanceName) + assert.Equal(t, 2, utils[0].AllocatedVcpus) + assert.Equal(t, int64(1024*1024*1024), utils[0].AllocatedMemoryBytes) + // All metrics should be 0 since we couldn't read proc + assert.Equal(t, uint64(0), utils[0].CPUUsec) + assert.Equal(t, uint64(0), utils[0].MemoryRSSBytes) +} + +func TestCollectVMUtilization_WithCurrentProcess(t *testing.T) { + // Test with current process PID to verify proc reading works + manager := &Manager{} + pid := os.Getpid() + + mockSource := &mockUtilizationSource{ + instances: []InstanceUtilizationInfo{ + { + ID: "test-instance", + Name: "test-vm", + HypervisorPID: &pid, + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB + }, + }, + } + manager.SetUtilizationSource(mockSource) + + utils, err := manager.CollectVMUtilization(context.Background()) + require.NoError(t, err) + require.Len(t, utils, 1) + + // Should have non-zero values since we're reading from current process + assert.True(t, utils[0].CPUUsec > 0 || utils[0].MemoryRSSBytes > 0, "should have some metrics") + assert.True(t, utils[0].MemoryRSSBytes > 0, "RSS should be positive") + assert.True(t, utils[0].MemoryVMSBytes > 0, "VMS should be positive") +} + +func TestVMUtilization_MemoryRatio(t *testing.T) { + // Test memory utilization ratio calculation + util := VMUtilization{ + MemoryRSSBytes: 536870912, // 512MB actual RSS + AllocatedMemoryBytes: 1073741824, // 1GB allocated + } + + // Calculate ratio as the metrics code would + ratio := float64(util.MemoryRSSBytes) / float64(util.AllocatedMemoryBytes) + assert.InDelta(t, 0.5, ratio, 0.001) // Should be ~50% +} + +func TestUtilizationMetrics_Integration(t *testing.T) { + // Create a manager with mock utilization source + manager := &Manager{} + + // Create mock source with test data + mockSource := &mockUtilizationSource{ + instances: []InstanceUtilizationInfo{ + { + ID: "vm-001", + Name: "web-server", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB + }, + { + ID: "vm-002", + Name: "database", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 8, + AllocatedMemoryBytes: 16 * 1024 * 1024 * 1024, // 16GB + }, + }, + } + manager.SetUtilizationSource(mockSource) + + // Collect utilization + utils, err := manager.CollectVMUtilization(context.Background()) + require.NoError(t, err) + require.Len(t, utils, 2) + + // Verify instance data is passed through + assert.Equal(t, "vm-001", utils[0].InstanceID) + assert.Equal(t, "web-server", utils[0].InstanceName) + assert.Equal(t, 4, utils[0].AllocatedVcpus) + + assert.Equal(t, "vm-002", utils[1].InstanceID) + assert.Equal(t, "database", utils[1].InstanceName) + assert.Equal(t, 8, utils[1].AllocatedVcpus) +} From 157b8fc618503c2f3ce5b7083987fca5ca87cf05 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:09:19 -0500 Subject: [PATCH 2/7] refactor: use OpenAPI spec for stats endpoint - Add InstanceStats schema and /instances/{id}/stats endpoint to openapi.yaml - Regenerate oapi code with make oapi-generate - Move stats implementation to instances.go following existing patterns - Remove custom stats.go and route from main.go --- cmd/api/api/instances.go | 77 ++++++ cmd/api/api/stats.go | 135 ---------- cmd/api/main.go | 11 - lib/oapi/oapi.go | 563 +++++++++++++++++++++++++++++---------- openapi.yaml | 91 +++++++ 5 files changed, 586 insertions(+), 291 deletions(-) delete mode 100644 cmd/api/api/stats.go diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e0cd9c74..3264309d 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -274,6 +274,83 @@ func (s *ApiService) GetInstance(ctx context.Context, request oapi.GetInstanceRe return oapi.GetInstance200JSONResponse(instanceToOAPI(*inst)), nil } +// GetInstanceStats returns resource utilization statistics for an instance +// The id parameter can be an instance ID, name, or ID prefix +// Note: Resolution is handled by ResolveResource middleware +func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInstanceStatsRequestObject) (oapi.GetInstanceStatsResponseObject, error) { + log := logger.FromContext(ctx) + inst := mw.GetResolvedInstance[instances.Instance](ctx) + if inst == nil { + return oapi.GetInstanceStats500JSONResponse{ + Code: "internal_error", + Message: "resource not resolved", + }, nil + } + + // Build stats response + stats := oapi.InstanceStats{ + InstanceId: inst.Id, + InstanceName: inst.Name, + AllocatedVcpus: inst.Vcpus, + AllocatedMemoryBytes: inst.Size + inst.HotplugSize, + } + + // Read /proc stats if we have a hypervisor PID + if inst.HypervisorPID != nil { + pid := *inst.HypervisorPID + + // Read CPU from /proc//stat + cpuUsec, err := resources.ReadProcStat(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc stat", "pid", pid, "error", err) + } else { + stats.CpuSeconds = float64(cpuUsec) / 1_000_000.0 + } + + // Read memory from /proc//statm + rssBytes, vmsBytes, err := resources.ReadProcStatm(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc statm", "pid", pid, "error", err) + } else { + stats.MemoryRssBytes = int64(rssBytes) + stats.MemoryVmsBytes = int64(vmsBytes) + + // Compute utilization ratio + if stats.AllocatedMemoryBytes > 0 { + ratio := float64(rssBytes) / float64(stats.AllocatedMemoryBytes) + stats.MemoryUtilizationRatio = &ratio + } + } + } + + // Read TAP stats if network is enabled + if inst.NetworkEnabled { + tapName := generateTAPName(inst.Id) + rxBytes, txBytes, err := resources.ReadTAPStats(tapName) + if err != nil { + log.DebugContext(ctx, "failed to read TAP stats", "tap", tapName, "error", err) + } else { + stats.NetworkRxBytes = int64(rxBytes) + stats.NetworkTxBytes = int64(txBytes) + } + } + + return oapi.GetInstanceStats200JSONResponse(stats), nil +} + +// generateTAPName generates TAP device name from instance ID +func generateTAPName(instanceID string) string { + // TAP name format: "hype-" + first 10 chars of instance ID + // Max TAP name length is 15 chars (IFNAMSIZ - 1) + prefix := "hype-" + maxIDLen := 15 - len(prefix) + idPart := instanceID + if len(idPart) > maxIDLen { + idPart = idPart[:maxIDLen] + } + return prefix + idPart +} + // DeleteInstance stops and deletes an instance // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware diff --git a/cmd/api/api/stats.go b/cmd/api/api/stats.go deleted file mode 100644 index 0675e42e..00000000 --- a/cmd/api/api/stats.go +++ /dev/null @@ -1,135 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http" - - "github.com/kernel/hypeman/lib/instances" - "github.com/kernel/hypeman/lib/logger" - mw "github.com/kernel/hypeman/lib/middleware" - "github.com/kernel/hypeman/lib/resources" -) - -// InstanceStats represents utilization statistics for a single instance -type InstanceStats struct { - InstanceID string `json:"instance_id"` - InstanceName string `json:"instance_name"` - - // CPU stats - CPUSeconds float64 `json:"cpu_seconds"` // Total CPU time consumed - - // Memory stats (from /proc//statm) - MemoryRSSBytes uint64 `json:"memory_rss_bytes"` // Resident Set Size - actual physical memory - MemoryVMSBytes uint64 `json:"memory_vms_bytes"` // Virtual Memory Size - total virtual memory - - // Network stats (from TAP interface) - NetworkRxBytes uint64 `json:"network_rx_bytes"` // Total bytes received - NetworkTxBytes uint64 `json:"network_tx_bytes"` // Total bytes transmitted - - // Allocated resources - AllocatedVcpus int `json:"allocated_vcpus"` - AllocatedMemoryBytes int64 `json:"allocated_memory_bytes"` - - // Utilization ratios - MemoryUtilizationRatio *float64 `json:"memory_utilization_ratio,omitempty"` // RSS / allocated -} - -// StatsHandler handles GET /instances/{id}/stats requests -func (s *ApiService) StatsHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := logger.FromContext(ctx) - - // Get resolved instance from context (set by ResolveResource middleware) - inst := mw.GetResolvedInstance[instances.Instance](ctx) - if inst == nil { - http.Error(w, `{"error": "instance not found"}`, http.StatusNotFound) - return - } - - // Build InstanceUtilizationInfo for this specific instance - info := resources.InstanceUtilizationInfo{ - ID: inst.Id, - Name: inst.Name, - } - - // Get hypervisor PID if running - if inst.HypervisorPID != nil { - info.HypervisorPID = inst.HypervisorPID - } - - // Get allocated resources - info.AllocatedVcpus = inst.Vcpus - - // Calculate allocated memory (Size + HotplugSize) - info.AllocatedMemoryBytes = inst.Size + inst.HotplugSize - - // Get TAP device if network enabled - if inst.NetworkEnabled { - info.TAPDevice = generateTAPName(inst.Id) - } - - // Collect stats directly for this instance - stats := InstanceStats{ - InstanceID: inst.Id, - InstanceName: inst.Name, - AllocatedVcpus: info.AllocatedVcpus, - AllocatedMemoryBytes: info.AllocatedMemoryBytes, - } - - // Read /proc stats if we have a PID - if info.HypervisorPID != nil { - pid := *info.HypervisorPID - - // Read CPU from /proc//stat - cpuUsec, err := resources.ReadProcStat(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc stat", "pid", pid, "error", err) - } else { - stats.CPUSeconds = float64(cpuUsec) / 1_000_000.0 - } - - // Read memory from /proc//statm - rssBytes, vmsBytes, err := resources.ReadProcStatm(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc statm", "pid", pid, "error", err) - } else { - stats.MemoryRSSBytes = rssBytes - stats.MemoryVMSBytes = vmsBytes - - // Compute utilization ratio - if info.AllocatedMemoryBytes > 0 { - ratio := float64(rssBytes) / float64(info.AllocatedMemoryBytes) - stats.MemoryUtilizationRatio = &ratio - } - } - } - - // Read TAP stats if we have a TAP device - if info.TAPDevice != "" { - rxBytes, txBytes, err := resources.ReadTAPStats(info.TAPDevice) - if err != nil { - log.DebugContext(ctx, "failed to read TAP stats", "tap", info.TAPDevice, "error", err) - } else { - stats.NetworkRxBytes = rxBytes - stats.NetworkTxBytes = txBytes - } - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(stats); err != nil { - log.ErrorContext(ctx, "failed to encode stats response", "error", err) - } -} - -// generateTAPName generates TAP device name from instance ID (same logic as instances package) -func generateTAPName(instanceID string) string { - // TAP name format: "hype-" + first 8 chars of instance ID - // Max TAP name length is 15 chars (IFNAMSIZ - 1) - prefix := "hype-" - maxIDLen := 15 - len(prefix) // 10 chars available for ID - idPart := instanceID - if len(idPart) > maxIDLen { - idPart = idPart[:maxIDLen] - } - return prefix + idPart -} diff --git a/cmd/api/main.go b/cmd/api/main.go index e336cf09..39587e91 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -282,17 +282,6 @@ func run() error { mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), ).Get("/instances/{id}/cp", app.ApiService.CpHandler) - // Instance stats endpoint (outside OpenAPI spec) - r.With( - middleware.RequestID, - middleware.RealIP, - middleware.Recoverer, - mw.InjectLogger(logger), - mw.AccessLogger(accessLogger), - mw.JwtAuth(app.Config.JwtSecret), - mw.ResolveResource(app.ApiService.NewResolvers(), api.ResolverErrorResponder), - ).Get("/instances/{id}/stats", app.ApiService.StatsHandler) - // OCI Distribution registry endpoints for image push (outside OpenAPI spec) r.Route("/v2", func(r chi.Router) { r.Use(middleware.RequestID) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 90d86f75..5b439652 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -621,6 +621,39 @@ type InstanceGPU struct { // - Unknown: Failed to determine state (see state_error for details) type InstanceState string +// InstanceStats Real-time resource utilization statistics for a VM instance +type InstanceStats struct { + // AllocatedMemoryBytes Total memory allocated to the VM (Size + HotplugSize) in bytes + AllocatedMemoryBytes int64 `json:"allocated_memory_bytes"` + + // AllocatedVcpus Number of vCPUs allocated to the VM + AllocatedVcpus int `json:"allocated_vcpus"` + + // CpuSeconds Total CPU time consumed by the VM hypervisor process in seconds + CpuSeconds float64 `json:"cpu_seconds"` + + // InstanceId Instance identifier + InstanceId string `json:"instance_id"` + + // InstanceName Instance name + InstanceName string `json:"instance_name"` + + // MemoryRssBytes Resident Set Size - actual physical memory used by the VM in bytes + MemoryRssBytes int64 `json:"memory_rss_bytes"` + + // MemoryUtilizationRatio Memory utilization ratio (RSS / allocated memory). Only present when allocated_memory_bytes > 0. + MemoryUtilizationRatio *float64 `json:"memory_utilization_ratio"` + + // MemoryVmsBytes Virtual Memory Size - total virtual memory allocated in bytes + MemoryVmsBytes int64 `json:"memory_vms_bytes"` + + // NetworkRxBytes Total network bytes received by the VM (from TAP interface) + NetworkRxBytes int64 `json:"network_rx_bytes"` + + // NetworkTxBytes Total network bytes transmitted by the VM (from TAP interface) + NetworkTxBytes int64 `json:"network_tx_bytes"` +} + // PassthroughDevice Physical GPU available for passthrough type PassthroughDevice struct { // Available Whether this GPU is available (not attached to an instance) @@ -1026,6 +1059,9 @@ type ClientInterface interface { // StatInstancePath request StatInstancePath(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetInstanceStats request + GetInstanceStats(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StopInstance request StopInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1439,6 +1475,18 @@ func (c *Client) StatInstancePath(ctx context.Context, id string, params *StatIn return c.Client.Do(req) } +func (c *Client) GetInstanceStats(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetInstanceStatsRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) StopInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewStopInstanceRequest(c.Server, id) if err != nil { @@ -2591,6 +2639,40 @@ func NewStatInstancePathRequest(server string, id string, params *StatInstancePa return req, nil } +// NewGetInstanceStatsRequest generates requests for GetInstanceStats +func NewGetInstanceStatsRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/instances/%s/stats", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewStopInstanceRequest generates requests for StopInstance func NewStopInstanceRequest(server string, id string) (*http.Request, error) { var err error @@ -3017,6 +3099,9 @@ type ClientWithResponsesInterface interface { // StatInstancePathWithResponse request StatInstancePathWithResponse(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*StatInstancePathResponse, error) + // GetInstanceStatsWithResponse request + GetInstanceStatsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceStatsResponse, error) + // StopInstanceWithResponse request StopInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StopInstanceResponse, error) @@ -3726,6 +3811,30 @@ func (r StatInstancePathResponse) StatusCode() int { return 0 } +type GetInstanceStatsResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *InstanceStats + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetInstanceStatsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetInstanceStatsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type StopInstanceResponse struct { Body []byte HTTPResponse *http.Response @@ -4205,6 +4314,15 @@ func (c *ClientWithResponses) StatInstancePathWithResponse(ctx context.Context, return ParseStatInstancePathResponse(rsp) } +// GetInstanceStatsWithResponse request returning *GetInstanceStatsResponse +func (c *ClientWithResponses) GetInstanceStatsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceStatsResponse, error) { + rsp, err := c.GetInstanceStats(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetInstanceStatsResponse(rsp) +} + // StopInstanceWithResponse request returning *StopInstanceResponse func (c *ClientWithResponses) StopInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StopInstanceResponse, error) { rsp, err := c.StopInstance(ctx, id, reqEditors...) @@ -5469,6 +5587,46 @@ func ParseStatInstancePathResponse(rsp *http.Response) (*StatInstancePathRespons return response, nil } +// ParseGetInstanceStatsResponse parses an HTTP response from a GetInstanceStatsWithResponse call +func ParseGetInstanceStatsResponse(rsp *http.Response) (*GetInstanceStatsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetInstanceStatsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest InstanceStats + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseStopInstanceResponse parses an HTTP response from a StopInstanceWithResponse call func ParseStopInstanceResponse(rsp *http.Response) (*StopInstanceResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -5896,6 +6054,9 @@ type ServerInterface interface { // Get filesystem path info // (GET /instances/{id}/stat) StatInstancePath(w http.ResponseWriter, r *http.Request, id string, params StatInstancePathParams) + // Get instance resource utilization stats + // (GET /instances/{id}/stats) + GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) StopInstance(w http.ResponseWriter, r *http.Request, id string) @@ -6094,6 +6255,12 @@ func (_ Unimplemented) StatInstancePath(w http.ResponseWriter, r *http.Request, w.WriteHeader(http.StatusNotImplemented) } +// Get instance resource utilization stats +// (GET /instances/{id}/stats) +func (_ Unimplemented) GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) func (_ Unimplemented) StopInstance(w http.ResponseWriter, r *http.Request, id string) { @@ -6945,6 +7112,37 @@ func (siw *ServerInterfaceWrapper) StatInstancePath(w http.ResponseWriter, r *ht handler.ServeHTTP(w, r) } +// GetInstanceStats operation middleware +func (siw *ServerInterfaceWrapper) GetInstanceStats(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetInstanceStats(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // StopInstance operation middleware func (siw *ServerInterfaceWrapper) StopInstance(w http.ResponseWriter, r *http.Request) { @@ -7375,6 +7573,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}/stat", wrapper.StatInstancePath) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/instances/{id}/stats", wrapper.GetInstanceStats) + }) r.Group(func(r chi.Router) { r.Post(options.BaseURL+"/instances/{id}/stop", wrapper.StopInstance) }) @@ -8529,6 +8730,41 @@ func (response StatInstancePath500JSONResponse) VisitStatInstancePathResponse(w return json.NewEncoder(w).Encode(response) } +type GetInstanceStatsRequestObject struct { + Id string `json:"id"` +} + +type GetInstanceStatsResponseObject interface { + VisitGetInstanceStatsResponse(w http.ResponseWriter) error +} + +type GetInstanceStats200JSONResponse InstanceStats + +func (response GetInstanceStats200JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceStats404JSONResponse Error + +func (response GetInstanceStats404JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetInstanceStats500JSONResponse Error + +func (response GetInstanceStats500JSONResponse) VisitGetInstanceStatsResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type StopInstanceRequestObject struct { Id string `json:"id"` } @@ -8932,6 +9168,9 @@ type StrictServerInterface interface { // Get filesystem path info // (GET /instances/{id}/stat) StatInstancePath(ctx context.Context, request StatInstancePathRequestObject) (StatInstancePathResponseObject, error) + // Get instance resource utilization stats + // (GET /instances/{id}/stats) + GetInstanceStats(ctx context.Context, request GetInstanceStatsRequestObject) (GetInstanceStatsResponseObject, error) // Stop instance (graceful shutdown) // (POST /instances/{id}/stop) StopInstance(ctx context.Context, request StopInstanceRequestObject) (StopInstanceResponseObject, error) @@ -9729,6 +9968,32 @@ func (sh *strictHandler) StatInstancePath(w http.ResponseWriter, r *http.Request } } +// GetInstanceStats operation middleware +func (sh *strictHandler) GetInstanceStats(w http.ResponseWriter, r *http.Request, id string) { + var request GetInstanceStatsRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetInstanceStats(ctx, request.(GetInstanceStatsRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetInstanceStats") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetInstanceStatsResponseObject); ok { + if err := validResponse.VisitGetInstanceStatsResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // StopInstance operation middleware func (sh *strictHandler) StopInstance(w http.ResponseWriter, r *http.Request, id string) { var request StopInstanceRequestObject @@ -9961,151 +10226,159 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XITubboq6j6nl3HOdt2nA+Y4FNTt0ICTPYhkEtI9j17wjVyt2xr0i31SGoHQ/F3", - "HmAecZ7klpak/rLa7gAx5MCuXTUmrc+lpfWttT4EIU9SzghTMhh+CGQ4IwmGn4dK4XB2yeMsIa/I7xmR", - "Sv85FTwlQlECjRKeMTVKsZrpf0VEhoKminIWDIMzrGboZkYEQXMYBckZz+IIjQmCfiQKugF5h5M0JsEw", - "2E6Y2o6wwkE3UItU/0kqQdk0+NgNBMERZ/HCTDPBWayC4QTHknRr057qoRGWSHfpQZ98vDHnMcEs+Agj", - "/p5RQaJg+Gt5G2/yxnz8GwmVnvxwjmmMxzE5JnMakmUwhJkQhKlRJOiciGVQHJnv8QKNecYiZNqhDsvi", - "GNEJYpyRrQow2JxGVENCN9FTB0MlMuKBTARrGtHIcwJHJ8h8RifHqDMj76qT7P40Pgiah2Q4IcuD/pIl", - "mPU0cPWy3PjQtjz2833fyJQnSTaaCp6lyyOfvDw9vUDwEbEsGRNRHvFgNx+PMkWmROgB05COcBQJIqV/", - "/+5jeW2DwWAwxLvDwaA/8K1yTljERSNIzWc/SHcGEVkxZCuQ2vGXQPri8uT45BAdcZFygaHv0kw1xC6D", - "p7yvMtpUT8WH/48zGkcerOd6YYpEI6yWNwWdkG1DOUOKJkQqnKRBN5hwkehOQYQV6ekvbVA9FASvmU63", - "aDXZMtJnBqajRDaN7pogylBC45hKEnIWyfIclKmH+82bKaEuEYJ7aMUT/WeUECnxlKCOJmCaijIkFVaZ", - "RFSiCaYxibbagMyHw2Yzv/ExohFhik5o9aYFY92gh8fhzu6e9xYneEpGEZ1anlAd/hj+jvgE6XEUgtb+", - "jWiUX7TbB0wpyGR5vqdARGESQSZEEBZ+9nSp4HPCMDPE/t9g3uB/bRfMcttyym0A5lnR/GM3+D0jGRml", - "XFKzwiUaYr9oNAJQI+jhXzN8WnXWJYySCovV9wNafIGbaNbXCjbnpmmdMgHhscNUbnYjAXoyJ0z5qBBT", - "9kN1x8/5FMWUEWRbWPhOuEB6gp9jPt0KvszeukEB0uULrdf9CQTJ/KFhNP2tGxCWJRqYMZ+WoTkjWKgx", - "qQCzgUHYgYrVNYL/rHIlqmcwxpKMVlOFM8oYiZBuaS+raYkyCXLg0vbhZlxTNZoTIb33CJb1X1Qh26Jx", - "qJiH1xMak9EMy5lZMY4iuIM4PqvsxCMLVYRLnGrC5gYEHi2R4uj8l8PdBw+RncADQ8kzEZoVLO+k1FsP", - "b9oihcUYx7EXN5rR7fZ8dxlD/Bhwnl+MJn6SY6BDTEO9AnuaevhukGZyZn4BPdarAn6myYBGr1j/fuPZ", - "9BEQCSODN2okfgnrZWoOG01jrmG6QBmjv2cV8bWPTrQkrpAm/jQiURdh+KDJMM4U700JI0LTKTQRPEFq", - "RlBJxEQd0p/2u+hKS109LWP28G5vMOgNroKqkBjv96ZppkGBlSJCL/D//Yp77w97/xr0Hr0pfo76vTd/", - "/zcfArSVezU66XXafXbc3e8it9iyMFxf6GpBeYWs6aMi5vhO9N2/7ekdnSwzeLP+iIfXRPQp347pWGCx", - "2GZTyt4NY6yIVNXdrG67dn+wthUbY1O99VturSb6A7p1Yn5DRKgpZUw0gsiuJpZUyS7CWnsEIoM0N/tP", - "FGKmcdYwdi4QYRG6oWqGMLSrQiBZ9HBKe9QsNegGCX73nLCpVt8f7i3ho0bGjv3Re/Mf7k9b/9uLkiKL", - "iQcZX/FMUTZF8Nlw3xmVqFgDVSRZy24ddLMYRKyEshPTbSdfCRYCL/yn5ha36vSk0sSn8fjMBfLs79gp", - "2BJZpQ0YAgbzCez32dnFtr6SKZZSzQTPprPyqfzq6MGbEiwapAG3yW4QUXk9onw0Tn1rovIanWy/RJpa", - "oZgmVBXUaWcwOH28La8C/Y8H7h9bfXRs7CqwfL15LizRlDMsCLDuCHGGjs4uEI5jHlplaKIlrAmdZoJE", - "/Zo2DKP7sIWw+Wfw4SdsTgVniZaF5lhQfXkqOv6H4MXL4yejJy8ug6E+ySgLrcJ89vLV62AY7A0Gg8DH", - "6vRJrEHGZ2cXR7Bj3X7GVRpn05Gk70nFOhXsPXsc1Bd+mO8XJSThwsijdgzUmVXJgWHXKKbXBF3p8cyh", - "7TyrE+pdmGoJaLNFSsScSp+e+Uv+TZ93Jkn5bprLUEUJScSciPys4fD7JV4fxjyLeqUpu8HvJAG0Lhbq", - "aeTX9VpxgTXkHccpZaSRvne/FZp8w8V1zHHU2/nCJJkRpcde3uIL86F6mBYBSH7+QXdJzmfRDY3UbBTx", - "G6aX7KE99gvKG+cE6J3eCY7/+uPPy9NCANl5Nk4tNdrZffCZ1KhGf/TQXuUi30iW+rdxkfo3cXn61x9/", - "up183U0QpvEzqhAdo69Xt/LPGVEzIkpcyR2w/pORDqE7cvhSmr5iACjbz5cIJ58TEeOFhxDuDDyU8J+C", - "Krhfth/SHA3pzmvIoB7NMa9lQjjwU0LPojxreqzvt6XLbVaSL2Rn99T+3G1Lm+dh6pQpu6Td+nJegBFc", - "i/BzKlSGY40nFTbntYkbb4tHLDDOnLJ4Ys8/xwesqibUtuKZGRlcL8vCil8iM1S+WSJb43nyGTZzLS/M", - "pOJJybyJOjUFjlZVveqJzXnci7DCQI9bMg2z3GWjfbIwQ5lDaULN0XTssQpoDKQMTekUjxeqKuDsDJaP", - "3g9oN74P1E0OLYMeJBop7vHTOGw5OdZwdG3b2A3B/TVSfDSfUM/IOaUqNFYqUVjznlmk1UP00pBab1oX", - "3cyopm0SOSAAQ7s8LQve/SvWQ3pxQ3ScT5APmw+pWTpYJ2CIDhelRVAwNKHxYgthdHnaR6/z1f67RAwr", - "OifOwzfDEo0JYSgDnkgimB/8luUFZFJrSFTVu1uZ3TgDt0C/4PZbH2kBLsEM3dA4BvtEghUNwbgxprX9", - "gFHZHJSeSRMAVoh5V6yMWdarWif5q90vr8iUSiVqzhfUefX0aG9v71GdSO8+6A12ejsPXu8MhgP9/3+1", - "99N8eX+nb6zDKr2w5qIyRTm6ODnetRyhOo96v48fHbx7h9Wjh/RGPnqfjMX0tz28EY+onzwdF3Yu1Mkk", - "ET1H+jRW+axbJSNSg/Xqk41St3LGOjP4KvZjdvdat7wL963PdWEN57d3sNaJ4FrnR2lzS/vRf9XyQYH5", - "JYXM2hhD6rWmHlN5/VgQfK1FeQ9/1exZjgzf8RsYMi28jheIvNNyLYmQ4FxNpFHSqmLKzv5P+wd7D/cP", - "BgOPr3QZiXlIR6HmKq0WoDXDGC+0cqr7oA5I1xEax3xcRd4Hew8Pfho82tltuw4jm7aDQy5FuV6oYyHy", - "dxcB475UFrW7+9PDvb29wcOHu/utVmUFvFaLcsJgRXT4ae+n/Z2D3f1WUPDJ+k+c77rui4s8SHqYpjE1", - "mk1PpiSkExoi8H4j3QF1EmBLJBezq3dyjKORsGKglx8oTGMPGEqmFjOZbYk6mqcnWaxoGhPzDQ6klaQL", - "Oz+GkXxmOcoYEaPctX+LkazHf605wu0lbwIiSkTG2XRq3CoF6E6pBMmiEIgoiaOhuaFr6RycZrGwN014", - "YPfQEhue8xsiejGZk7iMBIYd6cUmXBCU44k5tMquKJvjmEYjytLMixKNoHyaCZAvzaAIj3mmQJY0B1ae", - "BPwUoCNMNLlu5yYrDINLUz87u7ittSUVfEJjzzbmejD71bJ0Z4d4vj847+38HzA+vGTxwtAByhD0SXhE", - "+rUwLWjfentnTWvKY+RQeXVLe8KumccmlWu7DiISqZnWSDFDY4IsmzSWNLBTFpMUBP6Rj2BOBE7IOJtM", - "iBglHk3rqf6OTAOj+FOGTh9XiaYmzm3FrbPK4YC8NcEhZdOt1tD3aHK1bXRL0HzjP65XxPiSm1y3+qiE", - "bWO9t330Io9KRM/OLiTKZ+l7VLyWXpKz2UJq5cSMaCIxKCtrZoCcrcnwWdHR6rAeYpx4CZC7CKgzn6YZ", - "XMPzV72Tl5fbSUTm3cqa9MebGY+JXvdWSbaaOwdu4dKp2LnnTSKyQQzZ9gKVYJXf4NZAKt1XD3QUVzge", - "yZgrz2pe648IPqLO5VPjuNMr6KK0cpT67yUoVPD7offGaIrUNO05TFjXtSsXfK3ZIzFsq7y9yqS+q/IL", - "wbGJYa7icxEL5A6eX1cPml+vvb12EN+8J87VUeOciUd3OTo9NppZyJnClBGBEqKwjZguuRPBqx10g54W", - "BiJMEs4Qn0z+c7WDscF2k6PLKu3/aCns8k40/4bQIk3k4jmJUIIZnRCpbGhRZWY5w7sPHg5NUGNEJvsP", - "Hvb7fb9ZXYlFyqkvpuxJ/q3dUWwbp1SvGLMvZ593DnfgOG2zlw/B2eHrX4JhsJ1JsR3zEMfbckzZsPTv", - "/J/FB/hh/jmmzOtwbRUHSydL8a+V4001zzJ/H+qdMBLmCMlBSlxrm/Rz8hcaNWP6nkTIG4ai8BRp+Rsw", - "7vPiTT4jcrR4SKBKEaNlN0GL6FH6frW67QQjaGPnzJiicRFYu6xof1JotFwZabYUZZYSlseWxbH5FXI2", - "17fCF2hWIeDu29Jh3HBxTdl0FFEPdv7TfEQRFSRU4Mdff4eCbZym61HRL/zlNK1t0KwNmfFwl69OyT/F", - "4Fqd/eX0H7//X3n20287vz+/vPzv+bN/HL+g/30Zn738LD//6miprxrytNKnBlbGSqhTW/Q4xSr0CD4z", - "LlUD1OwXpDhKdOc+OgIFbXjFeug5VUTgeIiuApzSvgVmP+TJVYA65B0OlemFOEN6KDQjOCJiS3c+M7EO", - "uvMHpwN+rI8RLRhOaIiEBXLuQ5fZOOIJpmzril0xOxZyG5HgtNG/IhTiVGWC6BPRsma8QGOBtbpp1ehi", - "8i76gNP049YVA02UvFNC7yDFQuWhlW4GOGi7KuMUss1JhOY4zoi0muwVy/kHqOZ6EIXFlKh+bkIEQ03N", - "MdMAFK+awYWq+JYPBl3POSLdTh9kTKUiDOVWCSoBeVHHRQYcDCrX/2BwsN7/mOPQCvQD7F5+VuiQssX9", - "MAgMUxtiPJopla5/Jwj0xtwR9Mvr12caDPq/58gNVMAiP2KjjOE0jSmRxqumYpBJbDDGVuDznJnTbbmh", - "16ax7hbL9ft4AhOj18/PkSIioczQ706owTmhod4f+HeolJlGRYrR4dHpk61+i3eRANt8/SvO8XW+w5ob", - "wRm3ljVM6FEYzTV8u+jkuKvFKXtDC0EL/KZPuUCxITDFvR6iC0mqUQxwVMbFY04yXhQWMkPVr4ItN2Ja", - "pxRD9CqX73C+lDzku0AGN2RxL2HYK/ZPjRjGqbs0ere6VnBXW/3FkjZw4WKFrNEbWHEzKVh9/T0QhzvP", - "Wd32eLu7XTZa6sn8qFGc/Z1LIHu31SVvGz5bjQQqRX7lEbRfN/T1UwJZ3Qk9O7uAcFEsR5LhVM64ag7O", - "wMi1QeQdlUouB462CidYDpytsicTErsiGutLhsCKjDGIjKhv44sHt37NWINvL7B2ZSjs58azWgHtjsJZ", - "GwmCLxS0ShvMn79sYOqdLKcSYuojBmU+5gLBPjmqtBtQTxDMoZR0ykiETs6Kp1aFwcMNX9vTo93+zsOD", - "/s5g0N8ZtDH/JDhcMffp4VH7yQe7RiEe4vEwjIZk8hnmJ4vYRuDA8Q1eSHTlRMKrwMigJeGzdG2t2NjK", - "tbccvPtpsbp1JrguGvc20bet6P2qN9Dn1dfPreWKB//6rIfSpC0bPofGrtfoNoZRgkKexRH7d4XG+uYZ", - "VYBEVmORRBUPy+GyXrBrxm9YdevGPqbv7+8ZEQt0eXpasaYKMrFvbFtsnKdp4znw9FbHsLtGvFu7mlKw", - "9SYCrOuUsMSBvng4ddn04+I6DNa1MAEV4p/XTUqZAbc++xV7qinvEZmPsswn6OhPLkLz4uLkuHLgGD/c", - "ORgcPOodjHce9vajwU4P7+w97O0+wIPJXvjTXkM2ivZhEp8e+VC9oc0R0QB4MISZIPZoqO9QHrowzhTK", - "Xwbpy3mkJUZUkkNN/C/opq+MSKpHAO4a6i/xIhdVV3Y+w/qiur4p/Gt1j/NZprQYBH3kLFNI/wuWrLdg", - "Rf3VQ5g7P0QvOPSxK+1qRlnTGUxzzKLxYrl5Xb/o2AgQQaTigkQwmSVgQ/Q0J1o52bNkriOJ/WloqY2U", - "giiwLaNQW/HenlbQDSzUg25gQBh0AwcZ/dPsEH7B4oNuYBfiDbJcDlNYGRlRhFrU/eq3CaQpouiphFFp", - "KYYDdTQCli9zKRJ8q42M5hdU9DxNGYv0VWsb47I6pOUMq9kJm/Blk8JtmKV1FDoDTqqRRkIGiYgwSiIX", - "O5VzTYuH4HqMJUFRRizkDF4JbAGOjVklxWoGFx06UjatBl0tTdiGhZk1rH4zAfPahm2kbel3br0WGcDK", - "KMMS4cLN1Uqzp3Lkp8jLAwsyzWIsUD2Oa8WS5SKJKbtuM7pcJGMe0xDpDnVRaMLjmN+M9Cf5M+xlq9Xu", - "dIdRYdGtiTZmcdaebw6kNm+xhZ/1LrdqHsJQyyHbpv82pKRro7x4I52easZnQp0uGH1XQvRqAPL+7qDJ", - "IdwwaMUVvBwm1yamuHz3Lcr6bryLYDvMX0d6zInG0lXj/kcQu2a6VaMFvGFJYBFc5f7Ohyr5wJ0+5AK8", - "q3AtBVq3iut2ZNj7dCGXMRpcoivSdblh/ZT7pGw2r9tj5ok/wFYraU3QOjUq3DK8KlbmBwePHu3tP3i0", - "2wo0Vs/ODTUNhtsmY41bwbYkYe0hcvXEdh8M4H+3WpQx1fiX1GCuqS6o8qj4kxf0ccX1KQJAa2JEfj9W", - "JK0sTtLFilaOcv+gFbRWSCyHFbGnlFuiQyYTAoLvyMCtVyym5pBstYYQpzikauHx2eMb8NGgvEktkLHF", - "6LXFekBqx0Z4orTeOSdCZuMikr/jJkf/gcCGWcOFg9aPVmQ2HsEIHnNvfVZoZ52aUU35LdRtno3jkivF", - "PkfLM1v5LPg3OTDRDZYVq4T+HSoSdUu5Q+rmK9OifWo0h+t5drTCMO8LxvVnQisff+04u0GZmxToXIf4", - "KjbWfAU1VwaPaRsDgYcreiJ8w/UeoBp9sHzw03qNxuXnZCvf61XenrVOurI8rWFEt19uyZFwm471BzKA", - "VnYNFnLF2N3KyfqQwph5ml5fJy4pc+39DDXJNW2QNCo1Rh2SpGrhAgGdkrd1O7PTYT6gF6e+sOt28OhL", - "BI9drIwW+x/ynr9s6XOTrLXxLZ1pY4iGX+o8rnvRjHpl3zNWvT61V1pSrcghuypzuEnhDbqTDY+aZvV4", - "7ltkC2/Sloub45LDunTh65TAhmAI89i3tLPSSprPxph5PzO1OpUup/ongsxqMuvjjYypTOuSvfqDV/Nc", - "RlBQjSyADGA1CHJtd1mlXu19OsXv8hlA8cQS1fKymH2Ucpw9ewxP4F65h4904oaAZdQz7Dz+vJzzDquW", - "D2NVEnrnSPBePEt/VlC0prtVQ85iju7qPPeadJEwE1QtzjVDsD5yggURh5lBQ+AUsAn4czE5xNx9/Aja", - "5sQjdD4jjAgaosOzE8CSBDM81Ud2eYpiOiHhIoyJDZlaMjHDi7+XRyc9E+vpYgTAY00VAMQlwzg8O4F3", - "+DZlbDDo7/YhzxtPCcMpDYbBXn8HMg1oMMAWtyGUHn5am46+h8DJTiLLcR+bJhq0MuVMGuDsDga1FMS4", - "eOu8/Zs0xgrDXlvLdibb+rLbZykSyEkCdvkfu8H+YOdW61n7PNk37QXDmZpxQd8TWOaDWwLhkyY9YUY5", - "dlnoiG1Y4Gww/LWKrb+++fimG8gsSbAWEQ24ClilXDaJMEQijBi5sW8sfuPjPjo3qgW8VS7qWBjNn0Sa", - "JGGksOhP3yMswhmdkytmKbF5ao4FBJQmSFNgE85XRTMztTl9c4WJVI95tKhBNx9uWw8H0kgVwLfOzZzn", - "TUobkjT7qKNJzyBD7s1LQRhmqnjtb/IyXJMFSgWZ0HfekDwIM/Ibjo/zby6bd5W2a3GXsjDOooIBVrMo", - "e596SRIK4hOy/3H+8gWCiwf5mKFZER0FGbQo02QTRRlwHsCU/hV7gsMZMhQVkv9cBTS6Coq8u1tA/TJJ", - "DFHr9YAk/wwJyc00XRr93O/roQy1H6JfP5hRhugqYGkyUvyasKvgYxeVPkypmmXj/NubK+bdcIOufl6B", - "FeoYTN5y71n0DkuX2twCzCLELebEC4RRcUhlWX5MGRaLpiTWPFMjV0Wh4bmPbVbEoj8cDLbW25TtVj18", - "rtJQSwMfl8j67hejaJaaL1O0UsEKTT+YfcsVGTq+AZL6GEcuxPgH71jDO6zQW+IK0N9KDtsfaPTRoG9M", - "THxAjbRDXnNH2lMscEIUERLm9aEFhEZQ/W/nAQIl1aiAVeTtlsBTlwTfLCH2fmPC+Dz1OuDC/gbwD+Yt", - "MmzAvI82NS+OTX63vIjNvUJHOCyHiF2/2PqMqG8B4wabIqUuEdBXxN/7gj/PiJWEC6DVqNk2mTvzo9/P", - "rQTBibSjmMZaCD6HNfXOCVMISpXIvv2vk88gOuptzKdvh8iAMLaFWqRN7ZIbDzVTtLCETubla97PPggP", - "Z5hNiUQdwz//+uNPV2zirz/+tMUm/vrjT7ju27Z0EgyXl0l5O0T/RUjawzGdE7cZqbdA5kQs0N7AJvCF", - "T57n5fKKXbFXRGWCyTzmQ+8LYGIGhHc/DPZDWUYkkgBCSPw4scEIxjbh0Q3cXTag3OiN7i6pSHYHpQ1o", - "ruhwADxblFFFcYx4pkyOJlgHBJUWCzF7DsqT180sS4a39fRFkXfKYG/PLPCWBMaUGfLcO1N5x4yJOufn", - "T7b6CMR9gxUQcAJ6QzGM1QT6P2jSeppkKEqVoACUDW0qZRZqNNIc2zabsNI0ZR1qNtMISJFKtMbqNvND", - "7G5hsvHDzZlvfDaUY5cJs9mI8un79VUhaqVTfrlzdri3DHOb5rUA2dfQJlHHZujLH+JWcsl+LaTfCAEu", - "pSDOqTDi5vnvxjScI84mMQ0V6rm12Mo0udZTRZD7Qg5e2VUj7PZVj5Ius4rtSsBRI9OoFffdDPeoVxS+", - "BRspIrkLXPvBSdahzjGVIdd9S9jSC3FaqmEsi3taxqJ1tp1j+HvOclYK5nmtKFTU992QlcdOnbE6b9gA", - "UTyuEcSvSAhrz1RLbx/uEzZf5Kfosn6vMAJ9W6g52JwUtGmDkA/N75NFKKqBTVPBWZ4Yswm9bOrMOzxo", - "O4Nn4+dEuFttFmqeRxbbMl1ROCPhtdmQTU6/SiI4cfnr714OMPk/b8H97fJ/sPsWimMBq1XK4ol9M3t3", - "umKl5OmG3Y8WwTxABvf+uKgKTiLUwXLBwq3vygO5Ec5QTyZ/j27SWRbHzhA/J0IVWVDL9HT7g5YPWsjJ", - "7ratlEUuXj3vERZyCOYwoGsUSFzSwy8rLZsDM1v5gSZt9CsAlUOMZmH0M87fRHcWFZ//tvvU5iP62+5T", - "k5Hob3uHReHnu0GWwaZI86al13uMfFp4pVWgAWkyqQnXSXt5q40IfDYH7G1EvnyBP6S+NlJfGVwrBb88", - "He8din7VovAb9hPkyOaDNnxy8Wffmci3WdOTxchS4ZqKLd4ml4Ay4DazqC17cf8C5GiOcWX629KGWlzI", - "ldKBQ92T465NGmtSveaRxRuyqLp1bFxKtPNu3px6mIzpNOOZLOemhBzBRBYF1SoE+L7JrwV7bpRgv2Es", - "HWySdWxcQP2B93ckOtcP1BBvW4NtjfDsWm1GeC5cNe2lZ7fCH9JzK+m5BK7V0nOe/u8uxWczyVeTnx2+", - "+QBu375+jxL0fXu2wayNu+TsrdC41gJqkfJyNe8vCuRv3NGfT755udTlkbqf4afcBJxHThIseE2zKPit", - "4cNgs7Rv8yLgfUaxZ+VyJ35hy7y9iPl0/cuLfCT3zMDz9OKKudoob81zyLcoR1SkOJIkJqFCNzMazuAZ", - "hv4bjG9eaeA0fZu/u9waomcQ3ll+CQqTdyQRFMco5Ezy2OSCfTtPkrfD5bf8l6en0Mm8wDCv9t8OkXu/", - "n98xqVuVn1XoXcRYKvTCPhbp6AMXPI5NofO3Gp6l/W3ZBxfFE9Ur5nt8wciNHZBO0NvSO4y3DQ8xHBI+", - "16f0lW5+tzmTtdmL4kgA4EzdGgIVrH2PMGx1bc8TjJ2BNyVNy+cgZhl3/Bqku1xcfJq/+66gMk7Ttuhr", - "lwlYPE+SFTiMOkWtDyRVxDP1d6kiIkw5M4vdTciNOjg0/1D42hTfqlQfMRmKfaCyT5u9oApMiUGX2Nj8", - "a54kgSmFkmBfouLPf1ZTH3BZH9MnU3o784Nn3OZVTJXYl57F1DiHzZANWSC8ytsr0+C7l1xcKvGvjIab", - "d0WUVkGhHASLxgs42yJH+/16EwAHWewM+J3dl/eOuG+Nd8Smdv/u70iBH9/5LQm5gHqQ0tVnuT/BWyWN", - "o3TdO1AQoii00HVa7+Xp6VbTpTElBRuvjPihDts4yu+ep0CNjPt3W0x5KJxvYJWxUF8I1aijO521Uj9n", - "zDM9+lJaS6h9IBdSkcQo7JMshodtELVu8wPgcm2HLqJKQpbjLpisSnn9r9iYTDQ/TInQc+vukDar0D18", - "au25wvn1PTN38NvQayHTJahyWDVBbanEukty6dOd8rycn7ykp6CoVmtLSNSJ6bUpNoPmEsX6x9ZKTdcU", - "nvjS2Q8+/WblpVV8r1oNzubI/D1QuJMaWXNll+4dWXtGypfF0R84aB9Z4+kqNs/TH1ze1oD6IRPfS5kY", - "HD35bjpTgUPguNJW+fLLv7a8y/YH8+NknbtQ4XB26VKEfxus1GYUXjeN2+C9uJR2TxExT3o3fyd5nvT5", - "nj7b0IBzWwDTSdnx6ecCJpn894bdXz7GpQzHW0W4bPRuuefy38zd2jTns2tw4dpleNyXa24wze0EkheX", - "VVtRLk6zUqF1RUegUpLrllf56ZZLN5nsfLmCWhQLyKvE9K9YXhbHZQdER2cXXVsEugtlps0ItqhKH/mr", - "F0mEBXEljK6Y4ijEcZjFWBGUl/Expbdkg1v3Vam01Z3dt2ISz0Hn9YtkXqfmPukYfpyA0ysXwgGMK1VQ", - "bowttcWUNxJZapnZLeJK3Q5+hOC1iCotAatN2n3TvI/OszTlQkmkbjjUsZTgy4cki2MeLYYo78eQKX1k", - "SZytWWPzz5MI6obovqeVXPylAVzPVJBeylMgHZF5smNhbMSj5Sz/DYn8c/no7sJj66JD97a1AUprqZ5H", - "dY8oT7xvc8Fr2Fp4uSFaZXz3FTrJiw+EmVQ8ceOeHKMOzhTvTQnTwC3y/KeCz2lULxf3jdR4OsXvaJIl", - "eYHUZ4+h3KQwoR5QeBgCjRxOkXchIZGEyI+tW9aDWi4FZc/i03Lefzki5qhpo0z5FWOmi+SE+oi1jOmQ", - "XHGOYiymZOu7eZlo71rxMPHkuPYs8R5Ge88d9hVyRsv47nYqbUtN8y5iu3Nzx2Yjuy+/HS2slL/tHj4v", - "nOdiZlNI+beFgoPNsYRNh5Jf3mOrnda25jWwmQH0iD6Eec5DHKOIzEnMU6h4aNoG3SATsa3fNtze1mpa", - "rBW54cHgYBB8fPPx/wcAAP//PPEgqyLOAAA=", + "H4sIAAAAAAAC/+x9a3PbuJLoX0Hx7qmV9+hl+RFbW1O3HDvJ+Gyc+Maxz90zylUgEpIwJgEGAGUrqXyd", + "HzA/cX7JLbz4BCUqDyXeZGqqYpsk0Gh0N/qF7g+eT6OYEkQE94YfPO7PUQTVjydCQH9+Q8MkQq/QuwRx", + "If8cMxojJjBSL0U0IWIcQzGXvwWI+wzHAlPiDb1LKObgbo4YAgs1CuBzmoQBmCCgvkOB1/bQPYziEHlD", + "rxcR0QuggF7bE8tY/okLhsnM+9j2GIIBJeFSTzOFSSi84RSGHLVL017IoQHkQH7SUd+k400oDREk3kc1", + "4rsEMxR4w9/yy3iTvkwnvyNfyMlPFhCHcBKiM7TAPqqiwU8YQ0SMA4YXiFVRcaqfh0swoQkJgH4PtEgS", + "hgBPAaEE7RSQQRY4wBIT8hU5tTcULEEOzAQKpjEOHDtweg70Y3B+BlpzdF+cZPBocuTVD0lghKqD/ppE", + "kHQkciVYdnz1bn7s5/uukTGNomQ8YzSJqyOfv7y4uAbqISBJNEEsP+LRIB0PE4FmiMkBYx+PYRAwxLl7", + "/fZhHrZ+v98fwsGw3+/2XVAuEAkoq0WpfuxG6W4/QCuGbIRSM34FpS9uzs/OT8ApZTFlUH1bmalE2Hn0", + "5NeVJ5virrjo/3GCw8BB9VQCJlAwhqK6KPURMO9gSoDAEeICRrHX9qaURfIjL4ACdeSTJqTuMwTXTCff", + "aDRZlegTjdNxxOtGt68ATECEwxBz5FMS8PwcmIjD/frF5EgXMUYdsuKJ/DOIEOdwhkBLCjApRQngAoqE", + "A8zBFOIQBTtNUOaiYb2Y3+kE4AARgae4yGneRL7QgRN/d7Dn5OIIztA4wDNzJhSHP1N/B3QK5DgCqLfd", + "C5Ekv2y2DjUlQ9PqfE+VEFWTMDRFDBH/s6eLGV0gAokW9v+m5vX+Vy87LHvmpOwpZF5mr39se+8SlKBx", + "TDnWEFZkiHkiyUihGqgv3DCrR6v2OkdRXEC2mj/UG1+AEzV8jXBzpV8tSyYleMwwBc6uFUBPFogIlxQi", + "wjworvg5nYEQEwTMGwa/U8qAnOCXkM52vC+ztraXobTK0BLuTxBI+g81o8lnbQ+RJJLIDOksj805gkxM", + "UAGZNQeEGSiDrhb9lwWWKO7BBHI0Xi0VLjEhKADyTcOs+k2QcKUHVpavOOMWi/ECMe7kIwXWf2EBzBu1", + "Q4XUv53iEI3nkM81xDAIFA/C8LKwEocuVFAuYSwFmx1QndEcCAqufj0ZHBwCM4EDh5wmzNcQVFeS+1oO", + "r98FArIJDEMnbdST2+bnbpVC3BRwlTJG3XmSUqAlTC29PLObcvi2Fyd8rn9S8lhCpc4zKQYkeYXy5zeO", + "RZ8qIaF18FqLxK1hvYz1ZoNZSCVOlyAh+F1SUF+74Fxq4gJI4Y8DFLQBVA+kGIaJoJ0ZIohJOQWmjEZA", + "zBHIqZighbqzbhuMpNbVkTpmBw46/X6nP/KKSmK435nFiUQFFAIxCeD/+w123p90/tXvHL/Jfhx3O2/+", + "/m8uAmiq90pyknCadbYs77eBBTavDJcBXa0or9A1XVJEb9+55P1Nd+/0vHrAa/gD6t8i1sW0F+IJg2zZ", + "IzNM7ochFIiL4mpWv7t2fQq2FQsjM7n0DZdWUv0VubVCeoeYLyVliCSB8LYUlljwNoDSelRCBsjT7D+B", + "D4mkWX2wUwYQCcAdFnMA1XtFDETLDoxxB2tQvbYXwfvniMyk+X64V6FHSYwt80PnzX/YP+38bydJsiRE", + "DmJ8RROByQyox/r0nWMOMhiwQNHa49ZiNwmVihVhcq4/200hgYzBpXvXLHCrdo8LKXxqt08zkGN9Z9bA", + "5sAYbepAgMp9otb77PK6J1kyhpyLOaPJbJ7fld+sPHiTw0WNNmAX2fYCzG/HmI4nsQsmzG/Bee8lkNIK", + "hDjCIpNOu/3+xeMeH3nylwP7y04XnGm/igJfLp4yIzT5HDKkju4AUAJOL68BDEPqG2NoKjWsKZ4lDAXd", + "kjWsRndRCyKLzziHn5AFZpREUhdaQIYl8xRs/A/ei5dnT8ZPXtx4Q7mTQeIbg/ny5avX3tDb6/f7nuuo", + "kzuxhhifXV6fqhXL9+dUxGEyG3P8HhW8U97es8deGfCTdL0gQhFlWh81Y4DWvCgO9HENQnyLwEiOpzdt", + "91lZUA/UVBWkzZcxYgvMXXbmr+kzud8JR3ne1MxQJAmO2AKxdK/V5ndzZ70f0iTo5KZse+9QpMg6A9Tx", + "ktvWa3QKrBHvMIwxQbXyvf29yOQ7ym5DCoPO7hcWyQQJOXZ1iS/0g+JmGgJA6f577YqeT4I7HIj5OKB3", + "RILskD3mCUhfTgXQvVwJDP/648+bi0wB2X02iY002h0cfKY0KskfObTTuEgXksTuZVzH7kXcXPz1x592", + "Jd92EYhI+gwKQkfb68Wl/HOOxByx3KlkN1j+SWuH6nNg6SU3fcEBkPefVwQnXSAWwqVDEO72HZLwnwwL", + "xV/mOyBPNCA/XiMG5Wj28KoKwr5bEjqAcsD0WPK3kctNIEkB2R1cmB8HTWXzwo+tMWVAGpTBeaGc4FKF", + "X2AmEhhKOikcc06fuI62ONQCHczJqydm/1N6gKLoQm2qnumRVeilqqy4NTIt5es1sjWRJ5djM7Xy/IQL", + "GuXcm6BVMuBw0dQr7tiChp0ACqjkccNDQ4NbddpHSz2U3pQ60hzPJg6vgKRATMAMz+BkKYoKzm6/uvVu", + "RNvxXaiuC2hp8kDBWFBHnMZSy/mZxKN9t4nfUIW/xoKOF1PsGDmVVJnFijnwS9EzQ7RyiE7sYxNNa4O7", + "OZayjQOLBHWg3VzkFe/uiHSABG4IztIJ0mHTIeWRrrwTaogWZTkgsHI0gclyB0Bwc9EFr1No/50DAgVe", + "IBvhm0MOJggRkKgzEQVqfhW3zAOQcGkhYVH+3OjsOhi4o+wLap51gVTgIkjAHQ5D5Z+IoMC+cm5McGk9", + "yqmsN0rOJAUAydS8EclTlomqlkX+6vDLKzTDXLBS8AW0Xj093dvbOy4L6cFBp7/b2T14vdsf9uX//2oe", + "p/ny8U7XWCdFeWHcRXmJcnp9fjYwJ0JxHvF+Hx4f3d9DcXyI7/jx+2jCZr/vwa1ERN3i6Szzc4FWwhHr", + "WNEnqcrl3co5kWq8V5/slNooGGvd4KuOH7261/LNrxG+dYUujON88wBrWQiuDX7kFldZj/yr1A8yys8Z", + "ZMbH6GOnN/UM89vHDMFbqco7zld5PPOxPnfcDoZEKq+TJUD3Uq9FAWCUiinXRlpRTdndf7R/tHe4f9Tv", + "O2KlVSKmPh778lRpBIC0DEO4lMap/Aa0lHYdgElIJ0XiPdg7PHrUP94dNIVD66bN8JBqUfYr0DIY+bvN", + "gLFPCkANBo8O9/b2+oeHg/1GUBkFrxFQVhksqA6P9h7t7x4N9hthwaXrP7Gx63IsLnAQ6Ukch1hbNh0e", + "Ix9PsQ9U9BvID0ArUscSStXsIk9OYDBmRg10ngcC4tCBhpyrRU9m3gQteaZHSShwHCL9TG1II01XrfxM", + "jeRyy2FCEBunof0NRjIR/7XuCLuW9BWlogRoksxmOqySoe4Cc6VZZAoRRmEw1By6Vs6p3cwAe1NHB2YN", + "DanhOb1DrBOiBQrzRKCPIwlsRBkCKZ3oTSusCpMFDHEwxiROnCRRi8qnCVP6pR4UwAlNhNIl9YblJ1Fx", + "CmUjTKW4bhYmyxyDlamfXV5v6m2JGZ3i0LGMhRzMPDVHuvVDPN/vX3V2/49yPrwk4VLLAUyA+iaiAeqW", + "0rTU+42Xd1kHU5ojB/LQVdYE7WsOn1Rq7VqMcCDm0iKFBEwQMMek9qQpP2U2SSbgj10Cc8pghCbJdIrY", + "OHJYWk/lc6Bf0IY/JuDicVFoSuHcVN26LGyO0rem0MdkttMY+w5LrrSMdg6bb9zb9QrpWHJd6FZuFTPv", + "mOhtF7xIsxLBs8trDtJZug4Tr2GU5HK+5NI40SPqTAxM8paZIs7GYvgy+9DYsA5hHDkFkGUE0FrM4kSx", + "4dWrzvnLm14UoEW7AJN8eDenIZJw7+R0q4UN4GYhnYKfe1GnImvC4E0ZKIerlIMbIynHrw7sCCpgOOYh", + "FQ5oXsuHQD0ErZunOnAnIWiDuLCV8u85LBTo+9DJMVIi1U17pSYs29oFBl/r9oj0sZVfXmFSF6v8imCo", + "c5iL9JzlAtmNp7fFjaa3a7nXDOKa99yGOkonZ+SwXU4vzrRl5lMiICaIgQgJaDKmc+FEFdX22l5HKgMB", + "RBElgE6n/7k6wFjju0nJZZX1f1pJu/wqln9NapEUcuECBSCCBE8RFya1qDAzn8PBweFQJzUGaLp/cNjt", + "dt1udcGWMcWunLIn6bNmW9HTQalONmaXzz9vH75C4LTJWj54lyevf/WGXi/hrBdSH4Y9PsFkmPs9/TV7", + "oH7Qv04wcQZcG+XB4mkl/7WwvbE8s/Tfh3IlBPkpQVKlJa71TbpP8heSNEP8HgXAmYYi4AxI/VtR3Ofl", + "m3xG5mh2kUDkMkbzYYIG2aP4/Wpz2ypG6h0zZ0IEDrPE2qqh/Ump0XxlplklyyxGJM0tC0P9k0/JQnKF", + "K9GsIMDts8pm3FF2i8lsHGAHdf5TPwQBZsgXKo6/noe8Hozj9aToVv5SmdY0adakzDhOl28uyT/F4Vqc", + "/eXsH+/+L7989Pvuu+c3N/+9ePaPsxf4v2/Cy5efFedfnS31TVOeVsbUlJexkOrUlDwuoPAdis+cclGD", + "NfMECAoi+XEXnCoDbTgiHfAcC8RgOAQjD8a4a5DZ9Wk08kAL3UNf6K8AJUAOBeYIBojtyI8vda6D/PiD", + "tQE/lscIlgRG2AfMIDmNofNkEtAIYrIzIiNixgJ2IVwFbeRPAfBhLBKG5I5IXTNcggmD0tw0ZnQ2eRt8", + "gHH8cWdElCWK7gWTK4ghE2lqpZ1BbbSBSgeFzOsoAAsYJogbS3ZE0vNDmeZyEAHZDIlu6kJUjppSYKYG", + "KU4zgzJRiC0f9duOfQTyPbmRIeYCEZB6JTBXxAtaNjPgqF9g/6P+0fr4Y0pDK8hPUXf1WqElygb8oQlY", + "Ta2F8XguRLz+nqCSN5pHwK+vX19KNMh/r4AdKMNFusXaGINxHGLEdVRNhEonMckYO54rcqZ3t+GCXuuX", + "5WchX7+OJ2pi8Pr5FRCIRZho+d3yJTqn2JfrU/EdzHkiSRFDcHJ68WSn2+BepMJtCv+KfXydrrAURrDO", + "raqFqb7InOYSv21wftaW6pTh0EzRUnHTp5SBUAuYjK+H4JqjYhaD2iod4tE7GS4zD5mW6iNvx44YlyXF", + "ELxK9TuYgpKmfGfEYIfM+FINOyL/lIShg7qV0dtFWFW42tgvRrSpEC4UwDi91VFcLwpWs78D44rnKSn7", + "Hjfj7bzTUk7mJo1s77+6BrK3qS25afpsMRMol/mVZtB+29TXT0lktTv07PJapYtCPuYExnxORX1yBgT2", + "HYDuMRe8mjjaKJ2gmjhbPJ50SuyKbKwvmQLLEkJUZkR5GV88ufVb5hp8f4m1K1NhPzef1ShoXymdtVYg", + "uFJBi7JB//nLJqZ+FXAKKaYuYZA/x2wi2CdnlbY97EiCOeEczwgKwPlldtUqc3jY4UtrOh50dw+Purv9", + "fne338T9E0F/xdwXJ6fNJ+8PtEE8hJOhHwzR9DPcT4awtcIBwzu45GBkVcKRp3XQnPKZY1ujNjYK7VWT", + "dz8tV7d8CK7Lxt0k+7aRvF91B/qqePu5sV5x8K/PuiiNmh7DV+pl+9V4E8coAj5NwoD8uwATyXnaFECB", + "sVg4EtnFcsWs1+SW0DtSXLr2j0n+fZcgtgQ3FxcFbypDU3PHtsHCaRzX7gONN9qGwRr1bi00uWTrbSRY", + "lyVh7gT64unUedePzevQVNfABZSpf84wKSYa3XLvV6ypZLwHaDFOEpeiIx/ZDM3r6/OzwoZDeLh71D86", + "7hxNdg87+0F/twN39w47gwPYn+75j/ZqqlE0T5P49MyHIofWZ0QrxCtHmE5iD4aSh9LUhUkiQHozSDLn", + "qdQYQU4P1fm/yjZ9pVVSOYI6XX35JFymqurKjy+hZFT7bax+W/3F1TwRUg1S3/B5IoD8TYEsl2BU/dVD", + "aJ4fghdUfWMgbcuDsmQz6NchCSbL6utl+6JlMkAY4oIyFKjJjAAbgqep0ErFnhFzLY7Mj1qWmkwplQW2", + "ow1qo96b3fLansG61/Y0Cr22ZzEjf9QrVD8p4L22ZwBxJlnm6cblKEYwVDIsS8JIBA7xe81yEnTMBfa1", + "iQXVbtaxnbk+hIKxPkLrQjk6sm+O2fQjy9U3F6Cl7h38HRgLTP62k4Z98iy0PzjePz58NDg+bJS1mAG4", + "XhqfqryTKnBrRbMfJ2Nbladm6aeX1+rwkQcbTyKdJmnWntluUnD4UtvDBGRlfrLJj7vH+WTNgCaTMOdp", + "MNnaKiNQb5gzHzkVHDVxjnc4XODplLx7798Ofmc42r0/5IOJ0zhKJ3Jrkud571jF7EKTjr7f6c6nUwTF", + "eG3K6SvE1QrAFRJA0U9HCix5oqbpIobkbGKqwbiTsPb39vaOHh0MGtGVgS7HOGNl/1WhvDAQ5FhMvQla", + "r66uQC9HcHpMm0MXM8Tl4vQtCiefgVHS7+8h0C+k10nbY89FJTUKS0Y1ZuxFVIvyG6OxmEUZpKusl1Sb", + "qXC5E9t7e/1H+wdHB83Y2Fg8Y3a/WsKY90y0mCEf4UVh51vKo/r65BLI0dkU+kUNf3ewt39w+OhoI6jE", + "RlAJBgmPsBAbAXb06PBgf2+w2yx32uU1NbcCCgxblF0OpnMQhWM3HKioit523Wnh0hKrqXYrs/uydMFy", + "btgmyaDZTTDM1ag4l4cIWlKJyiukudtMO038DG4RKeepq7on1cWmeZqr0zIvoZifkymtusU3MfhMsosN", + "QsRS8eGqClKACEaBlV2p5Wd0KZU+E3IEggQZzGndiEGDcKhDAzEUc6Wsqg8xmRUThysTNjHDNAyr7/2p", + "ec2LTTxG3J2g8ZolClfaocsBzFI1GnmnMR+7rYrqwAzNkhAyUM5FXgEyX0YhJrdNRufLaEJD7AP5Qdmc", + "n9IwpHdj+Yj/otay02h18oNxFpUsmecaOBOT1htSmjdbwi9ylTulLBd18vf09z1VVrWJA86ZrftUGm86", + "Xfea4PscoRcv0ewP+nVJTTWDFtKZqqnem8p2Q7IujrdZ2CfpDX9HSExHa0oWbFEPLqzXtVoV1VqVwlXV", + "BEDL+vTsJaUiXnOXhRodxJ+l7q4oOfkZyu0iWqHU1mDroonmdHRwdHy8t39wPNhIR7HBhprgY13AwULQ", + "48gvFdMo6U0HffXfRkDpcIMbpJqQQxGgQmGMTwbo4wr2yS4x1Fi+qwovZztpTe2iEtxMzVyhsZwU1J5c", + "faQWmk6Rct6MNd46GTClpJpGMPgwhj4WS4cVBu9UngFIXykl4zcYvQSsA6VmbACnQlrrC8R4Msluo7Xs", + "5OA/tHVVooWjxhcveTKps+RelmfVdpxOzAlKXoIGRrqmCFcU+i5FJriDvOBZlz/7AgXtXP2rcghGv9G8", + "vKel9bTCZxZcdl0ocVfzzG9/aTtzmn9BUS1jfNUxVs+C8lRWWT9NnNyOU9FxS8Vfn8VQkg/mHPy0r8aT", + "/JXolXfOC/enGxcOq06rD6LNwc0Fwzf5sHzJU5GVgcFgLhu7XdhZF1HoUEVdBZHINhYo3QHFukC0uegD", + "ci+DFopisbTJ7NbI29ksdHKSDuikqS+cftQ//hIJ0NcrM57/h9SkyUer7CRr41SVPa1NM3RrnWflTBBt", + "Xpk7+cXMhdJNYy5W1EFf1f1Ct6FQtpNJ8Z0l5TtJG3S8qLOWM86xBc5ty4t1RuBK11RuZTlI6vdGhyo/", + "sz0I5rYvyCeizFgy63NmdbhH2pKdctEGfeWTYWUaGQRpxEoUpNZu1aRenUFxAe/TGZThCTko1RbT68jV", + "6Xz2WF3jfmUv7+OpHUKBUa4S9/jz+qZYqqpuxqpGKjYY7mQ8I39WSLQ63ioRZzZHe3WvFim6kJ8wLJZX", + "8kAweV4IMsROEk2G6qRQi1B/ziZXeeMfPyprc+pQOp8hghj2wcnluaKSCBI4k1t2cwFCPEX+0g+RSfut", + "hEnVrfWXp+cdfV/B5rmprCssFEJsQaeTy3NVS8aUPff63UFX1SqlMSIwxt7Q2+vuqmo5Eg1qiT11HUz9", + "aHw6kg/VSXYemBP3sX5FopbHlHCNnEG/XyqjD7N6Hb3fuXZW6OO1sW6nO4ZUUxcq2axWEzDgf2x7+/3d", + "jeBZW2LDNe01gYmYU4bfIwXmwYZI+KRJz4k2jm0lVWRezGjWG/5WpNbf3nx80/Z4EkVQqogaXRmuYsrr", + "VBjEAQQE3Zl7gr/TSRdcadNC1dvIejFpyx8FUiRBICDrzt4DyPw5XqARMZJYl0uBTF2KiICUwDolvUhm", + "emq9+5qFERePabAsYTcdrieHU9pIEcEb9xdIa//FNY0GXNJRlxjiPnXWVkIEEpFVrNG1hW6RigdO8b0z", + "rVylyrodx2fpM9uRoijbpbqLiR8mQXYAFjsBOK8rc+Qz5FKy/3H18gVQjKd6CqjXsgxfVQUSEyk2QZCo", + "k0dRSndEnkB/DrREVQXsRh4ORl5WO35HSb+EIy3UOh0lkn9RTTX0NG0c/NLtyqG0tB+C3z7oUYZg5JE4", + "Ggt6i8jI+9gGuQczLObJJH32ZkScC66x1a8KuAItTck79k6mXGGOqTUXQBIAaignXAIIsk3K6/ITTCBb", + "1jVioImozznQV1bNa9l9qsN+f2e9T9ks1XHOFV6U2sDHilgffDGJZqR5VaLlmi5J+UHMfeRAy/EtiNTH", + "MLDXZH6eHWvODqP05k4F9b3RHHofcPBRk2+IdI5bSbSr3hxWtMeQwQgJxLia10UWKr0Py99tBEgZqdoE", + "LBJvO4eesib4pkLY+7VNT9L2IYoW9rdAf2rerEqUmvd4W/PCUNcoTRuxPShyVJtlCbHtVlufIfE9UFx/", + "W6LUFrP7hvT7UOjnGTKacIa0kjTroYV1P7rj3IIhGHEzin5ZKsFXCqbOFSICqHZbvGv+tfqZyvB9G9LZ", + "2yHQKAxNszFuypOlzkN5KBpcqo909Yb0O1PUxJ9DMkMctPT5+dcff9qGSX/98adpmPTXH38qdu+Z9n9q", + "uLTV19sh+C+E4g4M8QLZxaisNbRAbAn2+qYIvXrkKJHCR2REXiGRMMLTnA+5LoUTPaC6u0rUejBJEAdc", + "oVAVL56aZATtm3DYBpaXNSq3ytHtiolkVpBbgDwVLQ2oyBYmWGAYApoIXWdQwaEuRmSA6DV7+cnLbpaK", + "4229fBHoXmjq7WgANxQwulWeg+909zg9JmhdXT3Z6QKl7muqUAknym7IhjGWQPenTFovk7REKQoUhWUt", + "m3LV8WqdNGfmnW14aeoq59W7aZgq842kxWoX81PtbuCycePNum9cPpQzW8253ony6et1ddJrZFN+uX22", + "tFfFuSlVnqHsW1iToGWqzKbFJAr10L8V0W9FAOfK6KdSGFBdwmJrFs4pJdMQ+wJ0LCymu1pq9RQJ5KGI", + "g1cGagDtuspZ0vmjoldIOKo9NEoN6rdzepS74m9wjGSZ3Bmt/TxJ1pHOGeY+ld/mqKXjwzjXh59nfJqn", + "onW+nTP19/TIWamYp/0OQdajfkteHjN1QspnwxaE4llJIH5DQVgqtZC7+/CQqPk63UXbuWKFE+j7Is3+", + "9rSgbTuEXGT+kDxCQQltUgrO0+LOdeRlyj9/xY02MzgWfoWY5WoNqL7iny1Lfwr8OfJv9YJMg5VVGsG5", + "7cHy9fUAXcN6g9PfgP/zuG9gOGa4WmUsnpu6D1/PViy07d5y+NEQmAPJKrw/sY5UXVIB8iXxd36oCORW", + "ToZyQ5QHxEmXSRhaR/wCMZFV8s7L094HqR800JMtt63URa5fPe8g4lOVzKFRV6uQ2MK9X1Zb1huml/KT", + "TJrYVwpVljDqldHP2H+d3QnSinp/Gzw1NfX+Nniqq+r9be9E19Xb+WrE0t+WaN629vqAiU8qr7iINCWa", + "dHndddpe+tZWFD5Tx3wTlS8F8KfW10Try6NrpeKXlpT/iqqfqdT9beIEKbG5sK0e2fyzH0zl267ryVBk", + "rvlawRdviktQllXHNq2bHl6CHE4pLi9/G/pQM4ZcqR1Y0j0/a5vC57pceZpZvCWPqoVj61qimXf77tST", + "aIJnCU14vr6yqnOPeNYUtCCAH5r+mh3PtRrsd0yl/W0eHVtXUH/S/VdSncsbqoW36SO6Rnm2b21Hec5C", + "Nc21ZwvhT+25kfacQ9dq7Tmtpfk11Wc9yTfTny29uRBu7r7+iBr0Q7u2QYyPOxfsLci4xgpqVj929dlv", + "aONbBPrTybevl9o6Ug8z/ZTqhPPAaoLZWVOvCn5v9NDfruzbvgr4kEnsWb5ll1vZ0ncvQjpbf/MiHcle", + "M3BcvRgR29/rrb4O+RakhAoEBRyFyBfgbo79ubqGIf+mxte3NGAcv03vXe4MwTOV3pm/Caomb3HEMAxV", + "UWga6nrmbxdR9HZYvct/c3GhPtI3MPSt/bdDYO/vpzzG5Vv5axVyFSHkArwwl0VacsMZDUNdgPatxGdu", + "fTvmwkV2RXVEXJcvCLozA+IpeJu7h/G25iKGJcLncpe+Eee36+t/67UICphCnO69hkhQcwlDYs19BWO3", + "7yxJ0/A6iAbjK98GqQDznM7Se98FUoZx3JR8DZiKihdRtIKGQStX85yLgCbi71wEiOmWnIa664gbtKCv", + "fxHwVjeQLHTQ0lX2XagyV5udqPJ0m1xbnF//togiT7fziqCr2P7nX6spD1i1x+TO5O7O/DwzNrkVUxT2", + "uWsxpZPDdHlQVSCcxtsr/cIPr7nYdhjfmAy3H4rIQYFVlwwSTJZqb7M+Iw/rToDayGxl6rwz63LyiH1W", + "yyOmPckPzyMZffzgXOJTpnoac9tj7OEkb+Usjhy7t1RTo6xZUNtavTcXFzt1TKPb4tayDPtpDps8yh/+", + "TFF9nh4et+gWhzBdwCpnoWQIUWujW5u10ANuQhM5eqWspep9wJdcoEgb7NMkVBfbVNa6qQ8A870d2gAL", + "rqoct5XLKlfXf0QmaCrPwxgxObf8XJXNymwPl1l7JWDKvpeaB78Pu1ZVulSmHBR1WCs1UIhjW+TSZTul", + "dTk/GaSnylAt9pbgoBXiW90wDSw4COUPOystXd144ktXP/h0zkpbq7hutWqaTYn5R5Bw5yWxZlsHPjix", + "9gzlmcXKH7XRbrHG18o1tmHvPYu7XA++7ohcIMHkO5Ah4NMwVOXetf7eixn1e6ovmB/jQDcIU8ApgVf/", + "OFIznl5eq/d0ae72iMhfqp2pyoDaBlfnvZdrfH+6J+H/YD1HL3AVW7g3/KdbZ/NQQC0P8RoWpfEqTZzG", + "PxVx02r0p9n6IM1WFYtNV9OaMegrpZibZrJuE9V0YOp90D+cr4voC+jPb2wV/+9D2zVFv9dNYxf4IJjS", + "rClA+tb99nmSpnXZH+jNKok4uwSlxORzE9yngO738KNR95dPQ8vjcaMktK3ylq1o8d3w1rZPPgODvVGR", + "x8dDYXNNaXYlqr543vvE8v2jVtpmti+QamaWqpa2rVU7311NF9BMfUhZP4+0kVN3RNLOVbaAp7Su2ta0", + "AgHmt3oEYz11gbvBmLbzTJexEREU+DD0kxAKBNJOW7o7Hq+xvl7lus99NX7LJnFsdNpijKetpB6SyeGm", + "CbV7+V5ViuKMOrUy/fvGvLON5G9zmG2Q+m1X8DNLtkHidw5ZTTpj6Ne74CqJY8oEB+KOqlazXKXbqDqo", + "ExoshyD9jgDdncyIONNWyrSIQIFq7SO/vSi0y8gNYL+MGerENFaiI9C36gyOtXpUbcRR02sj1Y++XgZ7", + "WXVob9q+IwdLcT+KawRpbwzTrkHi1uDLDtGoKYOrF1HaH8RPuKCRHff8DLRgImhnhohEbtaKI2Z0gYNy", + "R8fvpA3bBbzHURKlPYyfPVYdYZnOxlK9wVUuoKUpdO8jFHCVnLWzYcu2arc2sxef1pbiywkxK01rdcpv", + "eK0hqx8qt1jqmJbIBaUghGyGdn6Yy8OG17K7w+dnpZvDD/BCxsJSX6ZnNLyC0cykbWhpfo3rF6m7Y7uX", + "L26+HyssV2LxAd4AXqRqZt2tj++LBPvbOxK2fdvj5gF77aS1tSihTQ8gR3QRzHPqwxAEaIFCGqumpPpd", + "r+0lLDQtFoe9njTTQmnIDY/6R33v45uP/z8AAP//VH7pVYnYAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 1a6bc6ff..ee637a4a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -333,6 +333,60 @@ components: nullable: true example: "permission denied" + InstanceStats: + type: object + required: [instance_id, instance_name, cpu_seconds, memory_rss_bytes, memory_vms_bytes, network_rx_bytes, network_tx_bytes, allocated_vcpus, allocated_memory_bytes] + description: Real-time resource utilization statistics for a VM instance + properties: + instance_id: + type: string + description: Instance identifier + example: "qilviffnqzck2jrim1x6s2b1" + instance_name: + type: string + description: Instance name + example: "my-web-server" + cpu_seconds: + type: number + format: double + description: Total CPU time consumed by the VM hypervisor process in seconds + example: 29.94 + memory_rss_bytes: + type: integer + format: int64 + description: Resident Set Size - actual physical memory used by the VM in bytes + example: 443338752 + memory_vms_bytes: + type: integer + format: int64 + description: Virtual Memory Size - total virtual memory allocated in bytes + example: 4330745856 + network_rx_bytes: + type: integer + format: int64 + description: Total network bytes received by the VM (from TAP interface) + example: 12345678 + network_tx_bytes: + type: integer + format: int64 + description: Total network bytes transmitted by the VM (from TAP interface) + example: 87654321 + allocated_vcpus: + type: integer + description: Number of vCPUs allocated to the VM + example: 2 + allocated_memory_bytes: + type: integer + format: int64 + description: Total memory allocated to the VM (Size + HotplugSize) in bytes + example: 4294967296 + memory_utilization_ratio: + type: number + format: double + description: Memory utilization ratio (RSS / allocated memory). Only present when allocated_memory_bytes > 0. + nullable: true + example: 0.103 + CreateImageRequest: type: object required: [name] @@ -1577,6 +1631,43 @@ paths: schema: $ref: "#/components/schemas/Error" + /instances/{id}/stats: + get: + summary: Get instance resource utilization stats + description: | + Returns real-time resource utilization statistics for a running VM instance. + Metrics are collected from /proc//stat and /proc//statm for CPU and memory, + and from TAP interface statistics for network I/O. + operationId: getInstanceStats + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Instance ID or name + responses: + 200: + description: Instance utilization statistics + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceStats" + 404: + description: Instance not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /instances/{id}/volumes/{volumeId}: post: summary: Attach volume to instance From 58fc60438462cb2c3451c77bb1ab4fb028a5178f Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:20:12 -0500 Subject: [PATCH 3/7] fix: add ListRunningInstancesInfo to mock in builds tests --- lib/builds/manager_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index d47bfa34..7357ac8f 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -122,6 +122,10 @@ func (m *mockInstanceManager) ListInstanceAllocations(ctx context.Context) ([]re return nil, nil } +func (m *mockInstanceManager) ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) { + return nil, nil +} + // mockVolumeManager implements volumes.Manager for testing type mockVolumeManager struct { volumes map[string]*volumes.Volume From cafebe8e8fdb7532fc52c7e3e276bb46f2259974 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:45:44 -0500 Subject: [PATCH 4/7] fix: use os.Getpagesize() for correct memory metrics on ARM Replace hardcoded 4096 page size with os.Getpagesize() to support ARM systems (AWS Graviton, Apple Silicon) which may use 16KB or 64KB pages. Without this fix, memory metrics would be underreported by 4x-16x on non-x86 systems. --- lib/resources/utilization.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resources/utilization.go b/lib/resources/utilization.go index 15751d5d..d94b7366 100644 --- a/lib/resources/utilization.go +++ b/lib/resources/utilization.go @@ -190,8 +190,8 @@ func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { return 0, 0, fmt.Errorf("parse rss: %w", err) } - // Convert pages to bytes (page size is typically 4096) - const pageSize = 4096 + // Convert pages to bytes using system page size (varies by architecture) + pageSize := uint64(os.Getpagesize()) return rssPages * pageSize, vmsPages * pageSize, nil } From 125d603cec2e86e95d0b2d108dacb58b57bce390 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Fri, 23 Jan 2026 11:10:39 -0500 Subject: [PATCH 5/7] refactor: move VM metrics to dedicated lib/vm_metrics package This addresses PR review feedback: - Create new lib/vm_metrics package for better separation of concerns - Export GenerateTAPName from lib/network (fixes TAP name mismatch bug) - Simplify API handler to use vm_metrics.Manager - Add comprehensive README.md for the feature - Remove utilization code from lib/resources package The TAP name bug was causing network stats to always be zero because the API handler used different truncation logic (10 chars, no lowercase) than the canonical implementation (8 chars, lowercase). --- cmd/api/api/api.go | 40 ++-- cmd/api/api/instances.go | 88 +++----- cmd/api/wire.go | 31 +-- cmd/api/wire_gen.go | 65 +++--- lib/instances/manager.go | 14 +- lib/instances/manager_test.go | 35 --- lib/network/allocate.go | 7 +- lib/network/bridge.go | 2 +- lib/network/derive.go | 2 +- lib/network/manager_test.go | 2 +- lib/providers/providers.go | 21 +- lib/resources/resource.go | 45 ++-- lib/resources/utilization_metrics_test.go | 152 ------------- lib/resources/utilization_test.go | 194 ----------------- lib/vm_metrics/README.md | 130 +++++++++++ lib/vm_metrics/adapter.go | 92 ++++++++ .../collector.go} | 108 +--------- lib/vm_metrics/collector_test.go | 72 +++++++ lib/vm_metrics/manager.go | 143 +++++++++++++ lib/vm_metrics/manager_test.go | 202 ++++++++++++++++++ .../metrics.go} | 47 ++-- lib/vm_metrics/metrics_test.go | 75 +++++++ lib/vm_metrics/types.go | 61 ++++++ 23 files changed, 939 insertions(+), 689 deletions(-) delete mode 100644 lib/resources/utilization_metrics_test.go delete mode 100644 lib/resources/utilization_test.go create mode 100644 lib/vm_metrics/README.md create mode 100644 lib/vm_metrics/adapter.go rename lib/{resources/utilization.go => vm_metrics/collector.go} (51%) create mode 100644 lib/vm_metrics/collector_test.go create mode 100644 lib/vm_metrics/manager.go create mode 100644 lib/vm_metrics/manager_test.go rename lib/{resources/utilization_metrics.go => vm_metrics/metrics.go} (72%) create mode 100644 lib/vm_metrics/metrics_test.go create mode 100644 lib/vm_metrics/types.go diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 3cd61bb9..efb65eb1 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -10,20 +10,22 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" ) // ApiService implements the oapi.StrictServerInterface type ApiService struct { - Config *config.Config - ImageManager images.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager + Config *config.Config + ImageManager images.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager } var _ oapi.StrictServerInterface = (*ApiService)(nil) @@ -39,16 +41,18 @@ func New( ingressManager ingress.Manager, buildManager builds.Manager, resourceManager *resources.Manager, + vmMetricsManager *vm_metrics.Manager, ) *ApiService { return &ApiService{ - Config: config, - ImageManager: imageManager, - InstanceManager: instanceManager, - VolumeManager: volumeManager, - NetworkManager: networkManager, - DeviceManager: deviceManager, - IngressManager: ingressManager, - BuildManager: buildManager, - ResourceManager: resourceManager, + Config: config, + ImageManager: imageManager, + InstanceManager: instanceManager, + VolumeManager: volumeManager, + NetworkManager: networkManager, + DeviceManager: deviceManager, + IngressManager: ingressManager, + BuildManager: buildManager, + ResourceManager: resourceManager, + VMMetricsManager: vmMetricsManager, } } diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 3264309d..61280f39 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -278,7 +279,6 @@ func (s *ApiService) GetInstance(ctx context.Context, request oapi.GetInstanceRe // The id parameter can be an instance ID, name, or ID prefix // Note: Resolution is handled by ResolveResource middleware func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInstanceStatsRequestObject) (oapi.GetInstanceStatsResponseObject, error) { - log := logger.FromContext(ctx) inst := mw.GetResolvedInstance[instances.Instance](ctx) if inst == nil { return oapi.GetInstanceStats500JSONResponse{ @@ -287,68 +287,38 @@ func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInsta }, nil } - // Build stats response - stats := oapi.InstanceStats{ - InstanceId: inst.Id, - InstanceName: inst.Name, - AllocatedVcpus: inst.Vcpus, - AllocatedMemoryBytes: inst.Size + inst.HotplugSize, - } - - // Read /proc stats if we have a hypervisor PID - if inst.HypervisorPID != nil { - pid := *inst.HypervisorPID - - // Read CPU from /proc//stat - cpuUsec, err := resources.ReadProcStat(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc stat", "pid", pid, "error", err) - } else { - stats.CpuSeconds = float64(cpuUsec) / 1_000_000.0 - } + // Build instance info for metrics collection + info := vm_metrics.BuildInstanceInfo( + inst.Id, + inst.Name, + inst.HypervisorPID, + inst.NetworkEnabled, + inst.Vcpus, + inst.Size+inst.HotplugSize, + ) - // Read memory from /proc//statm - rssBytes, vmsBytes, err := resources.ReadProcStatm(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc statm", "pid", pid, "error", err) - } else { - stats.MemoryRssBytes = int64(rssBytes) - stats.MemoryVmsBytes = int64(vmsBytes) - - // Compute utilization ratio - if stats.AllocatedMemoryBytes > 0 { - ratio := float64(rssBytes) / float64(stats.AllocatedMemoryBytes) - stats.MemoryUtilizationRatio = &ratio - } - } - } + // Collect stats using vm_metrics manager + vmStats := s.VMMetricsManager.GetInstanceStats(ctx, info) - // Read TAP stats if network is enabled - if inst.NetworkEnabled { - tapName := generateTAPName(inst.Id) - rxBytes, txBytes, err := resources.ReadTAPStats(tapName) - if err != nil { - log.DebugContext(ctx, "failed to read TAP stats", "tap", tapName, "error", err) - } else { - stats.NetworkRxBytes = int64(rxBytes) - stats.NetworkTxBytes = int64(txBytes) - } - } - - return oapi.GetInstanceStats200JSONResponse(stats), nil + // Map domain type to API type + return oapi.GetInstanceStats200JSONResponse(vmStatsToOAPI(vmStats)), nil } -// generateTAPName generates TAP device name from instance ID -func generateTAPName(instanceID string) string { - // TAP name format: "hype-" + first 10 chars of instance ID - // Max TAP name length is 15 chars (IFNAMSIZ - 1) - prefix := "hype-" - maxIDLen := 15 - len(prefix) - idPart := instanceID - if len(idPart) > maxIDLen { - idPart = idPart[:maxIDLen] - } - return prefix + idPart +// vmStatsToOAPI converts vm_metrics.VMStats to oapi.InstanceStats +func vmStatsToOAPI(s *vm_metrics.VMStats) oapi.InstanceStats { + stats := oapi.InstanceStats{ + InstanceId: s.InstanceID, + InstanceName: s.InstanceName, + CpuSeconds: s.CPUSeconds(), + MemoryRssBytes: int64(s.MemoryRSSBytes), + MemoryVmsBytes: int64(s.MemoryVMSBytes), + NetworkRxBytes: int64(s.NetRxBytes), + NetworkTxBytes: int64(s.NetTxBytes), + AllocatedVcpus: s.AllocatedVcpus, + AllocatedMemoryBytes: s.AllocatedMemoryBytes, + MemoryUtilizationRatio: s.MemoryUtilizationRatio(), + } + return stats } // DeleteInstance stops and deletes an instance diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 6737b051..93b307de 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -19,25 +19,27 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" ) // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } // initializeApp is the injector function @@ -56,6 +58,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideIngressManager, providers.ProvideBuildManager, providers.ProvideResourceManager, + providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, wire.Struct(new(application), "*"), diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 5ac1bda9..c0a28717 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -20,6 +20,7 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" "log/slog" ) @@ -63,26 +64,31 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager) + if err != nil { + return nil, nil, err + } registry, err := providers.ProvideRegistry(paths, manager) if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager) + apiService := api.New(config, manager, instancesManager, volumesManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, vm_metricsManager) mainApplication := &application{ - Ctx: context, - Logger: logger, - Config: config, - ImageManager: manager, - SystemManager: systemManager, - NetworkManager: networkManager, - DeviceManager: devicesManager, - InstanceManager: instancesManager, - VolumeManager: volumesManager, - IngressManager: ingressManager, - BuildManager: buildsManager, - ResourceManager: resourcesManager, - Registry: registry, - ApiService: apiService, + Ctx: context, + Logger: logger, + Config: config, + ImageManager: manager, + SystemManager: systemManager, + NetworkManager: networkManager, + DeviceManager: devicesManager, + InstanceManager: instancesManager, + VolumeManager: volumesManager, + IngressManager: ingressManager, + BuildManager: buildsManager, + ResourceManager: resourcesManager, + VMMetricsManager: vm_metricsManager, + Registry: registry, + ApiService: apiService, } return mainApplication, func() { }, nil @@ -92,18 +98,19 @@ func initializeApp() (*application, func(), error) { // application struct to hold initialized components type application struct { - Ctx context.Context - Logger *slog.Logger - Config *config.Config - ImageManager images.Manager - SystemManager system.Manager - NetworkManager network.Manager - DeviceManager devices.Manager - InstanceManager instances.Manager - VolumeManager volumes.Manager - IngressManager ingress.Manager - BuildManager builds.Manager - ResourceManager *resources.Manager - Registry *registry.Registry - ApiService *api.ApiService + Ctx context.Context + Logger *slog.Logger + Config *config.Config + ImageManager images.Manager + SystemManager system.Manager + NetworkManager network.Manager + DeviceManager devices.Manager + InstanceManager instances.Manager + VolumeManager volumes.Manager + IngressManager ingress.Manager + BuildManager builds.Manager + ResourceManager *resources.Manager + VMMetricsManager *vm_metrics.Manager + Registry *registry.Registry + ApiService *api.ApiService } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 35c42c05..09c9616e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -3,7 +3,6 @@ package instances import ( "context" "fmt" - "strings" "sync" "github.com/kernel/hypeman/lib/devices" @@ -359,7 +358,7 @@ func (m *manager) ListRunningInstancesInfo(ctx context.Context) ([]resources.Ins // Derive TAP device name if networking is enabled if inst.NetworkEnabled { - info.TAPDevice = generateTAPName(inst.Id) + info.TAPDevice = network.GenerateTAPName(inst.Id) } infos = append(infos, info) @@ -368,14 +367,3 @@ func (m *manager) ListRunningInstancesInfo(ctx context.Context) ([]resources.Ins return infos, nil } -// generateTAPName generates TAP device name from instance ID. -// This matches the logic in network/allocate.go. -func generateTAPName(instanceID string) string { - // Use first 8 chars of instance ID - // hype-{8chars} fits within 15-char Linux interface name limit - shortID := instanceID - if len(shortID) > 8 { - shortID = shortID[:8] - } - return "hype-" + strings.ToLower(shortID) -} diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 1ee8ffee..3771bd17 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -1002,38 +1002,3 @@ func (r *testInstanceResolver) ResolveInstance(ctx context.Context, nameOrID str return nameOrID, nameOrID, nil } -func TestGenerateTAPName(t *testing.T) { - tests := []struct { - name string - instanceID string - expected string - }{ - { - name: "standard ID", - instanceID: "01HQVX7ABC123DEF456", - expected: "hype-01hqvx7a", - }, - { - name: "short ID", - instanceID: "ABC123", - expected: "hype-abc123", - }, - { - name: "exact 8 chars", - instanceID: "ABCDEFGH", - expected: "hype-abcdefgh", - }, - { - name: "mixed case", - instanceID: "AbCdEfGhIjKl", - expected: "hype-abcdefgh", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := generateTAPName(tt.instanceID) - assert.Equal(t, tt.expected, result) - }) - } -} diff --git a/lib/network/allocate.go b/lib/network/allocate.go index b622ace4..aaf36f82 100644 --- a/lib/network/allocate.go +++ b/lib/network/allocate.go @@ -53,7 +53,7 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N } // 5. Generate TAP name (tap-{first8chars-of-id}) - tap := generateTAPName(req.InstanceID) + tap := GenerateTAPName(req.InstanceID) // 6. Create TAP device with bidirectional rate limiting if err := m.createTAPDevice(tap, network.Bridge, network.Isolated, req.DownloadBps, req.UploadBps, req.UploadCeilBps); err != nil { @@ -269,8 +269,9 @@ func generateMAC() (string, error) { // TAPPrefix is the prefix used for hypeman TAP devices const TAPPrefix = "hype-" -// generateTAPName generates TAP device name from instance ID -func generateTAPName(instanceID string) string { +// GenerateTAPName generates TAP device name from instance ID. +// Exported for use by other packages (e.g., vm_metrics). +func GenerateTAPName(instanceID string) string { // Use first 8 chars of instance ID // hype-{8chars} fits within 15-char Linux interface name limit shortID := instanceID diff --git a/lib/network/bridge.go b/lib/network/bridge.go index ca150d8d..a979c111 100644 --- a/lib/network/bridge.go +++ b/lib/network/bridge.go @@ -803,7 +803,7 @@ func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs [] // Build set of expected TAP names for running instances expectedTAPs := make(map[string]bool) for _, id := range runningInstanceIDs { - tapName := generateTAPName(id) + tapName := GenerateTAPName(id) expectedTAPs[tapName] = true } diff --git a/lib/network/derive.go b/lib/network/derive.go index 591bab3e..01a81f9b 100644 --- a/lib/network/derive.go +++ b/lib/network/derive.go @@ -52,7 +52,7 @@ func (m *manager) deriveAllocation(ctx context.Context, instanceID string) (*All // 4. Use stored metadata to derive allocation (works for all hypervisors) if meta.IP != "" && meta.MAC != "" { - tap := generateTAPName(instanceID) + tap := GenerateTAPName(instanceID) // Determine state based on socket existence and snapshot socketPath := m.paths.InstanceSocket(instanceID, hypervisor.SocketNameForType(hypervisor.Type(meta.HypervisorType))) diff --git a/lib/network/manager_test.go b/lib/network/manager_test.go index b6c74ef3..9081528c 100644 --- a/lib/network/manager_test.go +++ b/lib/network/manager_test.go @@ -53,7 +53,7 @@ func TestGenerateTAPName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := generateTAPName(tt.instanceID) + got := GenerateTAPName(tt.instanceID) assert.Equal(t, tt.want, got) // Verify within Linux interface name limit (15 chars) assert.LessOrEqual(t, len(got), 15) diff --git a/lib/providers/providers.go b/lib/providers/providers.go index dc6c330e..9aede871 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -22,6 +22,7 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" "go.opentelemetry.io/otel" ) @@ -157,18 +158,26 @@ func ProvideResourceManager(ctx context.Context, cfg *config.Config, p *paths.Pa mgr.SetInstanceLister(instanceManager) mgr.SetVolumeLister(volumeManager) - // Set utilization source for VM metrics (instanceManager implements UtilizationSource) - mgr.SetUtilizationSource(instanceManager) - // Initialize resource discovery if err := mgr.Initialize(ctx); err != nil { return nil, fmt.Errorf("initialize resource manager: %w", err) } - // Initialize VM utilization metrics + return mgr, nil +} + +// ProvideVMMetricsManager provides the VM metrics manager for utilization tracking +func ProvideVMMetricsManager(instanceManager instances.Manager) (*vm_metrics.Manager, error) { + mgr := vm_metrics.NewManager() + + // Adapt instance manager to vm_metrics.InstanceSource interface + adapter := vm_metrics.NewInstanceManagerAdapter(instanceManager) + mgr.SetInstanceSource(adapter) + + // Initialize OTel metrics meter := otel.GetMeterProvider().Meter("hypeman") - if err := mgr.InitializeMetrics(meter); err != nil { - return nil, fmt.Errorf("initialize utilization metrics: %w", err) + if err := mgr.InitializeOTel(meter); err != nil { + return nil, fmt.Errorf("initialize vm metrics: %w", err) } return mgr, nil diff --git a/lib/resources/resource.go b/lib/resources/resource.go index c60c6e39..39e7e555 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -10,7 +10,6 @@ import ( "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/paths" - "go.opentelemetry.io/otel/metric" ) // ResourceType identifies a type of host resource. @@ -118,6 +117,19 @@ type VolumeLister interface { TotalVolumeBytes(ctx context.Context) (int64, error) } +// InstanceUtilizationInfo contains the minimal info needed to collect VM utilization metrics. +// Used by vm_metrics package via adapter. +type InstanceUtilizationInfo struct { + ID string + Name string + HypervisorPID *int // PID of the hypervisor process + TAPDevice string // Name of the TAP device (e.g., "hype-01234567") + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) +} + // Manager coordinates resource discovery and allocation tracking. type Manager struct { cfg *config.Config @@ -130,10 +142,6 @@ type Manager struct { instanceLister InstanceLister imageLister ImageLister volumeLister VolumeLister - - // Dependencies for utilization metrics - utilizationSource UtilizationSource - utilizationMetrics *UtilizationMetrics } // NewManager creates a new resource manager. @@ -166,33 +174,6 @@ func (m *Manager) SetVolumeLister(lister VolumeLister) { m.volumeLister = lister } -// SetUtilizationSource sets the utilization source for VM metrics collection. -func (m *Manager) SetUtilizationSource(source UtilizationSource) { - m.mu.Lock() - defer m.mu.Unlock() - m.utilizationSource = source -} - -// InitializeMetrics sets up OTel metrics for VM utilization. -// Must be called after SetUtilizationSource and before the manager is used. -// If meter is nil, metrics are not initialized. -func (m *Manager) InitializeMetrics(meter metric.Meter) error { - if meter == nil { - return nil - } - - metrics, err := newUtilizationMetrics(meter, m) - if err != nil { - return fmt.Errorf("initialize utilization metrics: %w", err) - } - - m.mu.Lock() - m.utilizationMetrics = metrics - m.mu.Unlock() - - return nil -} - // Initialize discovers host resources and registers them. // Must be called after setting listers and before using the manager. func (m *Manager) Initialize(ctx context.Context) error { diff --git a/lib/resources/utilization_metrics_test.go b/lib/resources/utilization_metrics_test.go deleted file mode 100644 index 1988bf31..00000000 --- a/lib/resources/utilization_metrics_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package resources - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/metric/metricdata" -) - -// mockUtilizationSourceWithData returns instances with simulated cgroup data -type mockUtilizationSourceWithData struct { - utilizations []VMUtilization -} - -func (m *mockUtilizationSourceWithData) ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) { - // Return info that will be used by CollectVMUtilization - // Since we can't easily mock the cgroup files in this test, we'll use a different approach - infos := make([]InstanceUtilizationInfo, len(m.utilizations)) - for i, u := range m.utilizations { - infos[i] = InstanceUtilizationInfo{ - ID: u.InstanceID, - Name: u.InstanceName, - AllocatedVcpus: u.AllocatedVcpus, - AllocatedMemoryBytes: u.AllocatedMemoryBytes, - } - } - return infos, nil -} - -func TestUtilizationMetrics_OTelIntegration(t *testing.T) { - // Create an in-memory metric reader - reader := metric.NewManualReader() - provider := metric.NewMeterProvider(metric.WithReader(reader)) - meter := provider.Meter("test") - - // Create manager - manager := &Manager{} - - // Initialize metrics - err := manager.InitializeMetrics(meter) - require.NoError(t, err) - - // Set up mock source - mockSource := &mockUtilizationSource{ - instances: []InstanceUtilizationInfo{ - { - ID: "test-vm-1", - Name: "web-app", - AllocatedVcpus: 2, - AllocatedMemoryBytes: 2 * 1024 * 1024 * 1024, - }, - }, - } - manager.SetUtilizationSource(mockSource) - - // Collect metrics - var rm metricdata.ResourceMetrics - err = reader.Collect(context.Background(), &rm) - require.NoError(t, err) - - // Verify we have scope metrics - require.NotEmpty(t, rm.ScopeMetrics, "should have scope metrics") - - // Find our metrics - var foundCPU, foundAllocVcpus, foundMemoryRSS, foundMemoryVMS, foundAllocMem, foundMemRatio bool - for _, sm := range rm.ScopeMetrics { - for _, m := range sm.Metrics { - switch m.Name { - case "hypeman_vm_cpu_seconds_total": - foundCPU = true - case "hypeman_vm_allocated_vcpus": - foundAllocVcpus = true - case "hypeman_vm_memory_rss_bytes": - foundMemoryRSS = true - case "hypeman_vm_memory_vms_bytes": - foundMemoryVMS = true - case "hypeman_vm_allocated_memory_bytes": - foundAllocMem = true - case "hypeman_vm_memory_utilization_ratio": - foundMemRatio = true - } - } - } - - assert.True(t, foundCPU, "should have CPU metric registered") - assert.True(t, foundAllocVcpus, "should have allocated vCPUs metric registered") - assert.True(t, foundMemoryRSS, "should have memory RSS metric registered") - assert.True(t, foundMemoryVMS, "should have memory VMS metric registered") - assert.True(t, foundAllocMem, "should have allocated memory metric registered") - assert.True(t, foundMemRatio, "should have memory utilization ratio metric registered") -} - -func TestUtilizationMetrics_NilMeter(t *testing.T) { - manager := &Manager{} - - // Should not error with nil meter - err := manager.InitializeMetrics(nil) - require.NoError(t, err) -} - -func TestUtilizationMetrics_MetricNames(t *testing.T) { - // Verify all expected metric names are correct when data is present - expectedMetrics := []string{ - "hypeman_vm_cpu_seconds_total", - "hypeman_vm_allocated_vcpus", - "hypeman_vm_memory_rss_bytes", - "hypeman_vm_memory_vms_bytes", - "hypeman_vm_allocated_memory_bytes", - "hypeman_vm_network_rx_bytes_total", - "hypeman_vm_network_tx_bytes_total", - "hypeman_vm_memory_utilization_ratio", - } - - reader := metric.NewManualReader() - provider := metric.NewMeterProvider(metric.WithReader(reader)) - meter := provider.Meter("test") - - manager := &Manager{} - // Use mock source with actual data so metrics will be emitted - manager.SetUtilizationSource(&mockUtilizationSource{ - instances: []InstanceUtilizationInfo{ - { - ID: "test-vm", - Name: "test", - AllocatedVcpus: 2, - AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB - }, - }, - }) - err := manager.InitializeMetrics(meter) - require.NoError(t, err) - - var rm metricdata.ResourceMetrics - err = reader.Collect(context.Background(), &rm) - require.NoError(t, err) - - // Collect all metric names - var metricNames []string - for _, sm := range rm.ScopeMetrics { - for _, m := range sm.Metrics { - metricNames = append(metricNames, m.Name) - } - } - - // Check all expected metrics are present - for _, expected := range expectedMetrics { - assert.Contains(t, metricNames, expected, "should have metric %s", expected) - } -} diff --git a/lib/resources/utilization_test.go b/lib/resources/utilization_test.go deleted file mode 100644 index 9435eb25..00000000 --- a/lib/resources/utilization_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package resources - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadProcStat(t *testing.T) { - // Test with current process - should work - pid := os.Getpid() - cpuUsec, err := ReadProcStat(pid) - require.NoError(t, err) - assert.True(t, cpuUsec >= 0, "CPU time should be non-negative") -} - -func TestReadProcStat_InvalidPID(t *testing.T) { - _, err := ReadProcStat(999999999) - assert.Error(t, err) -} - -func TestReadProcStatm(t *testing.T) { - // Test with current process - should work - pid := os.Getpid() - rssBytes, vmsBytes, err := ReadProcStatm(pid) - require.NoError(t, err) - assert.True(t, rssBytes > 0, "RSS should be positive") - assert.True(t, vmsBytes > 0, "VMS should be positive") - assert.True(t, vmsBytes >= rssBytes, "VMS should be >= RSS") -} - -func TestReadProcStatm_InvalidPID(t *testing.T) { - _, _, err := ReadProcStatm(999999999) - assert.Error(t, err) -} - -func TestReadTAPStats(t *testing.T) { - // This test requires /sys/class/net to exist - // We'll use loopback which should always exist - testInterface := "lo" - - basePath := filepath.Join("/sys/class/net", testInterface, "statistics") - if _, err := os.Stat(basePath); os.IsNotExist(err) { - t.Skip("skipping test: /sys/class/net not available") - } - - rxBytes, txBytes, err := ReadTAPStats(testInterface) - require.NoError(t, err) - // Loopback should have some traffic (or at least zero is valid) - assert.True(t, rxBytes >= 0 || txBytes >= 0, "should be able to read stats") -} - -func TestReadTAPStats_NotExists(t *testing.T) { - _, _, err := ReadTAPStats("nonexistent-tap-device") - assert.Error(t, err) -} - -// mockUtilizationSource implements UtilizationSource for testing -type mockUtilizationSource struct { - instances []InstanceUtilizationInfo -} - -func (m *mockUtilizationSource) ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) { - return m.instances, nil -} - -func TestCollectVMUtilization_WithMockSource(t *testing.T) { - // Create a manager with mock source - manager := &Manager{} - - // Test with nil source - should return nil, no error - utils, err := manager.CollectVMUtilization(context.Background()) - require.NoError(t, err) - assert.Nil(t, utils) - - // Set up mock source with no running instances - mockSource := &mockUtilizationSource{ - instances: []InstanceUtilizationInfo{}, - } - manager.SetUtilizationSource(mockSource) - - utils, err = manager.CollectVMUtilization(context.Background()) - require.NoError(t, err) - assert.Empty(t, utils) - - // Test with instances that have no PID (simulates instances where proc reading will fail gracefully) - mockSource.instances = []InstanceUtilizationInfo{ - { - ID: "test-instance-1", - Name: "test-vm", - HypervisorPID: nil, // No PID - proc reading skipped - TAPDevice: "", // No TAP - network reading skipped - AllocatedVcpus: 2, - AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB - }, - } - - utils, err = manager.CollectVMUtilization(context.Background()) - require.NoError(t, err) - require.Len(t, utils, 1) - assert.Equal(t, "test-instance-1", utils[0].InstanceID) - assert.Equal(t, "test-vm", utils[0].InstanceName) - assert.Equal(t, 2, utils[0].AllocatedVcpus) - assert.Equal(t, int64(1024*1024*1024), utils[0].AllocatedMemoryBytes) - // All metrics should be 0 since we couldn't read proc - assert.Equal(t, uint64(0), utils[0].CPUUsec) - assert.Equal(t, uint64(0), utils[0].MemoryRSSBytes) -} - -func TestCollectVMUtilization_WithCurrentProcess(t *testing.T) { - // Test with current process PID to verify proc reading works - manager := &Manager{} - pid := os.Getpid() - - mockSource := &mockUtilizationSource{ - instances: []InstanceUtilizationInfo{ - { - ID: "test-instance", - Name: "test-vm", - HypervisorPID: &pid, - AllocatedVcpus: 4, - AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB - }, - }, - } - manager.SetUtilizationSource(mockSource) - - utils, err := manager.CollectVMUtilization(context.Background()) - require.NoError(t, err) - require.Len(t, utils, 1) - - // Should have non-zero values since we're reading from current process - assert.True(t, utils[0].CPUUsec > 0 || utils[0].MemoryRSSBytes > 0, "should have some metrics") - assert.True(t, utils[0].MemoryRSSBytes > 0, "RSS should be positive") - assert.True(t, utils[0].MemoryVMSBytes > 0, "VMS should be positive") -} - -func TestVMUtilization_MemoryRatio(t *testing.T) { - // Test memory utilization ratio calculation - util := VMUtilization{ - MemoryRSSBytes: 536870912, // 512MB actual RSS - AllocatedMemoryBytes: 1073741824, // 1GB allocated - } - - // Calculate ratio as the metrics code would - ratio := float64(util.MemoryRSSBytes) / float64(util.AllocatedMemoryBytes) - assert.InDelta(t, 0.5, ratio, 0.001) // Should be ~50% -} - -func TestUtilizationMetrics_Integration(t *testing.T) { - // Create a manager with mock utilization source - manager := &Manager{} - - // Create mock source with test data - mockSource := &mockUtilizationSource{ - instances: []InstanceUtilizationInfo{ - { - ID: "vm-001", - Name: "web-server", - HypervisorPID: nil, - TAPDevice: "", - AllocatedVcpus: 4, - AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB - }, - { - ID: "vm-002", - Name: "database", - HypervisorPID: nil, - TAPDevice: "", - AllocatedVcpus: 8, - AllocatedMemoryBytes: 16 * 1024 * 1024 * 1024, // 16GB - }, - }, - } - manager.SetUtilizationSource(mockSource) - - // Collect utilization - utils, err := manager.CollectVMUtilization(context.Background()) - require.NoError(t, err) - require.Len(t, utils, 2) - - // Verify instance data is passed through - assert.Equal(t, "vm-001", utils[0].InstanceID) - assert.Equal(t, "web-server", utils[0].InstanceName) - assert.Equal(t, 4, utils[0].AllocatedVcpus) - - assert.Equal(t, "vm-002", utils[1].InstanceID) - assert.Equal(t, "database", utils[1].InstanceName) - assert.Equal(t, 8, utils[1].AllocatedVcpus) -} diff --git a/lib/vm_metrics/README.md b/lib/vm_metrics/README.md new file mode 100644 index 00000000..3257b84a --- /dev/null +++ b/lib/vm_metrics/README.md @@ -0,0 +1,130 @@ +# VM Metrics + +This package provides real-time resource utilization metrics for VMs managed by Hypeman. + +## Overview + +VM metrics are collected from the **host's perspective** by reading: +- `/proc//stat` - CPU time (user + system) for the hypervisor process +- `/proc//statm` - Memory usage (RSS and VMS) for the hypervisor process +- `/sys/class/net//statistics/` - Network I/O from TAP interfaces + +This approach works for both Cloud Hypervisor and QEMU without requiring any in-guest agents. + +## Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `hypeman_vm_cpu_seconds_total` | Counter | Total CPU time consumed by VM | +| `hypeman_vm_allocated_vcpus` | Gauge | Number of vCPUs allocated | +| `hypeman_vm_memory_rss_bytes` | Gauge | Resident Set Size (physical memory) | +| `hypeman_vm_memory_vms_bytes` | Gauge | Virtual Memory Size | +| `hypeman_vm_allocated_memory_bytes` | Gauge | Total allocated memory | +| `hypeman_vm_network_rx_bytes_total` | Counter | Network bytes received | +| `hypeman_vm_network_tx_bytes_total` | Counter | Network bytes transmitted | +| `hypeman_vm_memory_utilization_ratio` | Gauge | RSS / allocated memory | + +All metrics include `instance_id` and `instance_name` labels. + +## API Endpoint + +```bash +GET /instances/{id}/stats +``` + +Returns current utilization for a specific instance: + +```json +{ + "instance_id": "abc123", + "instance_name": "my-vm", + "cpu_seconds": 42.5, + "memory_rss_bytes": 536870912, + "memory_vms_bytes": 4294967296, + "network_rx_bytes": 1048576, + "network_tx_bytes": 524288, + "allocated_vcpus": 2, + "allocated_memory_bytes": 4294967296, + "memory_utilization_ratio": 0.125 +} +``` + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Host │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ /proc/ │ │ /proc/ │ │ /sys/class/ │ │ +│ │ /stat │ │ /statm │ │ net// │ │ +│ │ (CPU time) │ │ (memory) │ │ statistics/ │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ └───────────────────┼───────────────────┘ │ +│ │ │ +│ ┌────────▼────────┐ │ +│ │ vm_metrics │ │ +│ │ Manager │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ┌──────────────┼──────────────┐ │ +│ │ │ │ │ +│ ┌──────▼──────┐ ┌─────▼─────┐ ┌─────▼─────┐ │ +│ │ OTel/OTLP │ │ REST API │ │ Grafana │ │ +│ │ Exporter │ │ /stats │ │ Dashboard │ │ +│ └─────────────┘ └───────────┘ └───────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Limitations + +These metrics measure the **hypervisor process**, not the guest OS: + +- **CPU**: Time spent by the hypervisor process, not guest CPU utilization +- **Memory RSS**: Physical memory used by hypervisor, closely correlates with guest memory +- **Memory VMS**: Virtual address space of hypervisor process +- **Network**: Bytes through TAP interface (accurate for guest traffic) + +For detailed in-guest metrics (per-process CPU, filesystem usage, etc.), +consider running an exporter like Prometheus Node Exporter inside the guest. + +## Usage + +```go +// Create manager +mgr := vm_metrics.NewManager() + +// Set instance source (implements InstanceSource interface) +mgr.SetInstanceSource(instanceManager) + +// Initialize OTel metrics (optional) +meter := otel.GetMeterProvider().Meter("hypeman") +if err := mgr.InitializeOTel(meter); err != nil { + return err +} + +// Get stats for a specific instance +info := vm_metrics.BuildInstanceInfo( + inst.Id, + inst.Name, + inst.HypervisorPID, + inst.NetworkEnabled, + inst.Vcpus, + inst.Size + inst.HotplugSize, +) +stats := mgr.GetInstanceStats(ctx, info) +``` + +## Prometheus Queries + +```promql +# CPU utilization rate (per vCPU) +rate(hypeman_vm_cpu_seconds_total[1m]) / hypeman_vm_allocated_vcpus + +# Memory utilization percentage +hypeman_vm_memory_rss_bytes / hypeman_vm_allocated_memory_bytes * 100 + +# Network throughput (bytes/sec) +rate(hypeman_vm_network_rx_bytes_total[1m]) +rate(hypeman_vm_network_tx_bytes_total[1m]) +``` diff --git a/lib/vm_metrics/adapter.go b/lib/vm_metrics/adapter.go new file mode 100644 index 00000000..0f18a187 --- /dev/null +++ b/lib/vm_metrics/adapter.go @@ -0,0 +1,92 @@ +package vm_metrics + +import ( + "context" + + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/resources" +) + +// InstanceManagerAdapter adapts an instance manager that returns resources.InstanceUtilizationInfo +// to the vm_metrics.InstanceSource interface. +type InstanceManagerAdapter struct { + manager interface { + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) + } +} + +// NewInstanceManagerAdapter creates an adapter for the given instance manager. +func NewInstanceManagerAdapter(manager interface { + ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) +}) *InstanceManagerAdapter { + return &InstanceManagerAdapter{manager: manager} +} + +// ListRunningInstancesForMetrics implements InstanceSource. +func (a *InstanceManagerAdapter) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + ctx := context.Background() + infos, err := a.manager.ListRunningInstancesInfo(ctx) + if err != nil { + return nil, err + } + + result := make([]InstanceInfo, len(infos)) + for i, info := range infos { + result[i] = InstanceInfo{ + ID: info.ID, + Name: info.Name, + HypervisorPID: info.HypervisorPID, + TAPDevice: info.TAPDevice, + AllocatedVcpus: info.AllocatedVcpus, + AllocatedMemoryBytes: info.AllocatedMemoryBytes, + } + } + return result, nil +} + +// InstanceListerAdapter adapts an instance lister that provides Instance structs +// to the vm_metrics.InstanceSource interface. +// This is useful when you need to build InstanceInfo directly from Instance data. +type InstanceListerAdapter struct { + listFunc func(ctx context.Context) ([]InstanceData, error) +} + +// InstanceData contains the minimal instance data needed for metrics. +type InstanceData struct { + ID string + Name string + HypervisorPID *int + NetworkEnabled bool + AllocatedVcpus int + AllocatedMemoryBytes int64 +} + +// NewInstanceListerAdapter creates an adapter with a custom list function. +func NewInstanceListerAdapter(listFunc func(ctx context.Context) ([]InstanceData, error)) *InstanceListerAdapter { + return &InstanceListerAdapter{listFunc: listFunc} +} + +// ListRunningInstancesForMetrics implements InstanceSource. +func (a *InstanceListerAdapter) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + ctx := context.Background() + instances, err := a.listFunc(ctx) + if err != nil { + return nil, err + } + + result := make([]InstanceInfo, len(instances)) + for i, inst := range instances { + info := InstanceInfo{ + ID: inst.ID, + Name: inst.Name, + HypervisorPID: inst.HypervisorPID, + AllocatedVcpus: inst.AllocatedVcpus, + AllocatedMemoryBytes: inst.AllocatedMemoryBytes, + } + if inst.NetworkEnabled { + info.TAPDevice = network.GenerateTAPName(inst.ID) + } + result[i] = info + } + return result, nil +} diff --git a/lib/resources/utilization.go b/lib/vm_metrics/collector.go similarity index 51% rename from lib/resources/utilization.go rename to lib/vm_metrics/collector.go index d94b7366..e9149e10 100644 --- a/lib/resources/utilization.go +++ b/lib/vm_metrics/collector.go @@ -1,118 +1,13 @@ -// Package resources provides host resource discovery, capacity tracking, -// and oversubscription-aware allocation management for CPU, memory, disk, and network. -package resources +package vm_metrics import ( - "context" "fmt" "os" "path/filepath" "strconv" "strings" - - "github.com/kernel/hypeman/lib/logger" ) -// VMUtilization holds actual resource utilization metrics for a VM. -// These are real-time values read from /proc//stat, /proc//statm, and TAP interfaces. -type VMUtilization struct { - InstanceID string - InstanceName string - CPUUsec uint64 // Total CPU time in microseconds (user + system) - MemoryRSSBytes uint64 // Resident Set Size - actual physical memory used - MemoryVMSBytes uint64 // Virtual Memory Size - total allocated virtual memory - NetRxBytes uint64 // Total network bytes received - NetTxBytes uint64 // Total network bytes transmitted - - // Allocated resources (for computing utilization ratios) - AllocatedVcpus int // Number of allocated vCPUs - AllocatedMemoryBytes int64 // Allocated memory in bytes -} - -// UtilizationSource provides access to instance data for utilization collection. -type UtilizationSource interface { - // ListRunningInstancesInfo returns basic info for running instances. - ListRunningInstancesInfo(ctx context.Context) ([]InstanceUtilizationInfo, error) -} - -// InstanceUtilizationInfo contains the minimal info needed to collect utilization. -type InstanceUtilizationInfo struct { - ID string - Name string - HypervisorPID *int // PID of the hypervisor process - TAPDevice string // Name of the TAP device (e.g., "hype-01234567") - - // Allocated resources (for computing utilization ratios) - AllocatedVcpus int // Number of allocated vCPUs - AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) -} - -// CollectVMUtilization gathers utilization metrics for all running VMs. -// Uses /proc//stat and /proc//statm for per-process metrics (no cgroups needed). -func (m *Manager) CollectVMUtilization(ctx context.Context) ([]VMUtilization, error) { - m.mu.RLock() - source := m.utilizationSource - m.mu.RUnlock() - - if source == nil { - return nil, nil - } - - log := logger.FromContext(ctx) - - instances, err := source.ListRunningInstancesInfo(ctx) - if err != nil { - return nil, fmt.Errorf("list running instances: %w", err) - } - - var utilizations []VMUtilization - for _, inst := range instances { - util := VMUtilization{ - InstanceID: inst.ID, - InstanceName: inst.Name, - AllocatedVcpus: inst.AllocatedVcpus, - AllocatedMemoryBytes: inst.AllocatedMemoryBytes, - } - - // Collect per-process metrics from /proc if we have a PID - if inst.HypervisorPID != nil { - pid := *inst.HypervisorPID - - // Read CPU time from /proc//stat - cpuUsec, err := ReadProcStat(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc stat", "instance_id", inst.ID, "pid", pid, "error", err) - } else { - util.CPUUsec = cpuUsec - } - - // Read memory from /proc//statm - rssBytes, vmsBytes, err := ReadProcStatm(pid) - if err != nil { - log.DebugContext(ctx, "failed to read proc statm", "instance_id", inst.ID, "pid", pid, "error", err) - } else { - util.MemoryRSSBytes = rssBytes - util.MemoryVMSBytes = vmsBytes - } - } - - // Collect TAP stats if we have a TAP device - if inst.TAPDevice != "" { - rxBytes, txBytes, err := ReadTAPStats(inst.TAPDevice) - if err != nil { - log.DebugContext(ctx, "failed to read TAP stats", "instance_id", inst.ID, "tap", inst.TAPDevice, "error", err) - } else { - util.NetRxBytes = rxBytes - util.NetTxBytes = txBytes - } - } - - utilizations = append(utilizations, util) - } - - return utilizations, nil -} - // ReadProcStat reads CPU time from /proc//stat. // Returns total CPU time (user + system) in microseconds. // Fields 14 and 15 are utime and stime in clock ticks. @@ -191,6 +86,7 @@ func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { } // Convert pages to bytes using system page size (varies by architecture) + // x86_64: typically 4KB, ARM64: can be 4KB, 16KB, or 64KB pageSize := uint64(os.Getpagesize()) return rssPages * pageSize, vmsPages * pageSize, nil } diff --git a/lib/vm_metrics/collector_test.go b/lib/vm_metrics/collector_test.go new file mode 100644 index 00000000..6905a35b --- /dev/null +++ b/lib/vm_metrics/collector_test.go @@ -0,0 +1,72 @@ +package vm_metrics + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadProcStat(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + cpuUsec, err := ReadProcStat(pid) + require.NoError(t, err) + assert.True(t, cpuUsec >= 0, "CPU time should be non-negative") +} + +func TestReadProcStat_InvalidPID(t *testing.T) { + _, err := ReadProcStat(999999999) + assert.Error(t, err) +} + +func TestReadProcStatm(t *testing.T) { + // Test with current process - should work + pid := os.Getpid() + rssBytes, vmsBytes, err := ReadProcStatm(pid) + require.NoError(t, err) + assert.True(t, rssBytes > 0, "RSS should be positive") + assert.True(t, vmsBytes > 0, "VMS should be positive") + assert.True(t, vmsBytes >= rssBytes, "VMS should be >= RSS") +} + +func TestReadProcStatm_InvalidPID(t *testing.T) { + _, _, err := ReadProcStatm(999999999) + assert.Error(t, err) +} + +func TestReadProcStatm_PageSize(t *testing.T) { + // Verify that the page size is being used correctly + // The returned values should be multiples of the page size + pid := os.Getpid() + rssBytes, vmsBytes, err := ReadProcStatm(pid) + require.NoError(t, err) + + pageSize := uint64(os.Getpagesize()) + // RSS and VMS should be exact multiples of page size + assert.Equal(t, uint64(0), rssBytes%pageSize, "RSS should be a multiple of page size") + assert.Equal(t, uint64(0), vmsBytes%pageSize, "VMS should be a multiple of page size") +} + +func TestReadTAPStats(t *testing.T) { + // This test requires /sys/class/net to exist + // We'll use loopback which should always exist + testInterface := "lo" + + basePath := filepath.Join("/sys/class/net", testInterface, "statistics") + if _, err := os.Stat(basePath); os.IsNotExist(err) { + t.Skip("skipping test: /sys/class/net not available") + } + + rxBytes, txBytes, err := ReadTAPStats(testInterface) + require.NoError(t, err) + // Loopback should have some traffic (or at least zero is valid) + assert.True(t, rxBytes >= 0 || txBytes >= 0, "should be able to read stats") +} + +func TestReadTAPStats_NotExists(t *testing.T) { + _, _, err := ReadTAPStats("nonexistent-tap-device") + assert.Error(t, err) +} diff --git a/lib/vm_metrics/manager.go b/lib/vm_metrics/manager.go new file mode 100644 index 00000000..efdad42b --- /dev/null +++ b/lib/vm_metrics/manager.go @@ -0,0 +1,143 @@ +package vm_metrics + +import ( + "context" + "sync" + + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/network" + "go.opentelemetry.io/otel/metric" +) + +// Manager collects and exposes VM resource utilization metrics. +// It reads from /proc and TAP interfaces to gather real-time statistics. +type Manager struct { + mu sync.RWMutex + source InstanceSource + otel *otelMetrics +} + +// NewManager creates a new VM metrics manager. +func NewManager() *Manager { + return &Manager{} +} + +// SetInstanceSource sets the source for instance information. +// Must be called before collecting metrics. +func (m *Manager) SetInstanceSource(source InstanceSource) { + m.mu.Lock() + defer m.mu.Unlock() + m.source = source +} + +// InitializeOTel sets up OpenTelemetry metrics. +// If meter is nil, OTel metrics are disabled. +func (m *Manager) InitializeOTel(meter metric.Meter) error { + if meter == nil { + return nil + } + + otel, err := newOTelMetrics(meter, m) + if err != nil { + return err + } + + m.mu.Lock() + m.otel = otel + m.mu.Unlock() + + return nil +} + +// GetInstanceStats collects metrics for a single instance. +// Returns nil if the instance is not running or stats cannot be collected. +func (m *Manager) GetInstanceStats(ctx context.Context, info InstanceInfo) *VMStats { + log := logger.FromContext(ctx) + + stats := &VMStats{ + InstanceID: info.ID, + InstanceName: info.Name, + AllocatedVcpus: info.AllocatedVcpus, + AllocatedMemoryBytes: info.AllocatedMemoryBytes, + } + + // Read /proc stats if we have a hypervisor PID + if info.HypervisorPID != nil { + pid := *info.HypervisorPID + + // Read CPU from /proc//stat + cpuUsec, err := ReadProcStat(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc stat", "instance_id", info.ID, "pid", pid, "error", err) + } else { + stats.CPUUsec = cpuUsec + } + + // Read memory from /proc//statm + rssBytes, vmsBytes, err := ReadProcStatm(pid) + if err != nil { + log.DebugContext(ctx, "failed to read proc statm", "instance_id", info.ID, "pid", pid, "error", err) + } else { + stats.MemoryRSSBytes = rssBytes + stats.MemoryVMSBytes = vmsBytes + } + } + + // Read TAP stats if we have a TAP device + if info.TAPDevice != "" { + rxBytes, txBytes, err := ReadTAPStats(info.TAPDevice) + if err != nil { + log.DebugContext(ctx, "failed to read TAP stats", "instance_id", info.ID, "tap", info.TAPDevice, "error", err) + } else { + stats.NetRxBytes = rxBytes + stats.NetTxBytes = txBytes + } + } + + return stats +} + +// CollectAll gathers metrics for all running VMs. +// Used by OTel metrics callback. +func (m *Manager) CollectAll(ctx context.Context) ([]VMStats, error) { + m.mu.RLock() + source := m.source + m.mu.RUnlock() + + if source == nil { + return nil, nil + } + + instances, err := source.ListRunningInstancesForMetrics() + if err != nil { + return nil, err + } + + var stats []VMStats + for _, info := range instances { + s := m.GetInstanceStats(ctx, info) + if s != nil { + stats = append(stats, *s) + } + } + + return stats, nil +} + +// BuildInstanceInfo creates an InstanceInfo from instance metadata. +// This is a helper for the API layer to avoid duplicating TAP name logic. +func BuildInstanceInfo(id, name string, pid *int, networkEnabled bool, vcpus int, memoryBytes int64) InstanceInfo { + info := InstanceInfo{ + ID: id, + Name: name, + HypervisorPID: pid, + AllocatedVcpus: vcpus, + AllocatedMemoryBytes: memoryBytes, + } + + if networkEnabled { + info.TAPDevice = network.GenerateTAPName(id) + } + + return info +} diff --git a/lib/vm_metrics/manager_test.go b/lib/vm_metrics/manager_test.go new file mode 100644 index 00000000..851771e7 --- /dev/null +++ b/lib/vm_metrics/manager_test.go @@ -0,0 +1,202 @@ +package vm_metrics + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockInstanceSource implements InstanceSource for testing +type mockInstanceSource struct { + instances []InstanceInfo +} + +func (m *mockInstanceSource) ListRunningInstancesForMetrics() ([]InstanceInfo, error) { + return m.instances, nil +} + +func TestManager_GetInstanceStats(t *testing.T) { + mgr := NewManager() + + // Test with no PID - should return stats with zero values + info := InstanceInfo{ + ID: "test-instance-1", + Name: "test-vm", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 2, + AllocatedMemoryBytes: 1024 * 1024 * 1024, // 1GB + } + + stats := mgr.GetInstanceStats(context.Background(), info) + require.NotNil(t, stats) + assert.Equal(t, "test-instance-1", stats.InstanceID) + assert.Equal(t, "test-vm", stats.InstanceName) + assert.Equal(t, 2, stats.AllocatedVcpus) + assert.Equal(t, int64(1024*1024*1024), stats.AllocatedMemoryBytes) + assert.Equal(t, uint64(0), stats.CPUUsec) + assert.Equal(t, uint64(0), stats.MemoryRSSBytes) +} + +func TestManager_GetInstanceStats_WithCurrentProcess(t *testing.T) { + mgr := NewManager() + pid := os.Getpid() + + info := InstanceInfo{ + ID: "test-instance", + Name: "test-vm", + HypervisorPID: &pid, + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, // 4GB + } + + stats := mgr.GetInstanceStats(context.Background(), info) + require.NotNil(t, stats) + + // Should have non-zero values since we're reading from current process + assert.True(t, stats.CPUUsec >= 0, "CPU time should be non-negative") + assert.True(t, stats.MemoryRSSBytes > 0, "RSS should be positive") + assert.True(t, stats.MemoryVMSBytes > 0, "VMS should be positive") +} + +func TestManager_CollectAll_NilSource(t *testing.T) { + mgr := NewManager() + + // Test with nil source - should return nil, no error + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + assert.Nil(t, stats) +} + +func TestManager_CollectAll_EmptySource(t *testing.T) { + mgr := NewManager() + mgr.SetInstanceSource(&mockInstanceSource{instances: []InstanceInfo{}}) + + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + assert.Empty(t, stats) +} + +func TestManager_CollectAll_MultipleInstances(t *testing.T) { + mgr := NewManager() + mgr.SetInstanceSource(&mockInstanceSource{ + instances: []InstanceInfo{ + { + ID: "vm-001", + Name: "web-server", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 4, + AllocatedMemoryBytes: 4 * 1024 * 1024 * 1024, + }, + { + ID: "vm-002", + Name: "database", + HypervisorPID: nil, + TAPDevice: "", + AllocatedVcpus: 8, + AllocatedMemoryBytes: 16 * 1024 * 1024 * 1024, + }, + }, + }) + + stats, err := mgr.CollectAll(context.Background()) + require.NoError(t, err) + require.Len(t, stats, 2) + + assert.Equal(t, "vm-001", stats[0].InstanceID) + assert.Equal(t, "web-server", stats[0].InstanceName) + assert.Equal(t, 4, stats[0].AllocatedVcpus) + + assert.Equal(t, "vm-002", stats[1].InstanceID) + assert.Equal(t, "database", stats[1].InstanceName) + assert.Equal(t, 8, stats[1].AllocatedVcpus) +} + +func TestVMStats_CPUSeconds(t *testing.T) { + stats := &VMStats{ + CPUUsec: 1500000, // 1.5 seconds in microseconds + } + assert.InDelta(t, 1.5, stats.CPUSeconds(), 0.001) +} + +func TestVMStats_MemoryUtilizationRatio(t *testing.T) { + tests := []struct { + name string + rss uint64 + allocated int64 + expectRatio *float64 + expectNil bool + }{ + { + name: "normal ratio", + rss: 536870912, // 512MB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(0.5), + }, + { + name: "100% utilization", + rss: 1073741824, // 1GB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(1.0), + }, + { + name: "over 100% utilization", + rss: 2147483648, // 2GB + allocated: 1073741824, // 1GB + expectRatio: ptrFloat64(2.0), + }, + { + name: "zero allocated", + rss: 536870912, + allocated: 0, + expectNil: true, + }, + { + name: "negative allocated", + rss: 536870912, + allocated: -1, + expectNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stats := &VMStats{ + MemoryRSSBytes: tt.rss, + AllocatedMemoryBytes: tt.allocated, + } + ratio := stats.MemoryUtilizationRatio() + if tt.expectNil { + assert.Nil(t, ratio) + } else { + require.NotNil(t, ratio) + assert.InDelta(t, *tt.expectRatio, *ratio, 0.001) + } + }) + } +} + +func TestBuildInstanceInfo(t *testing.T) { + pid := 1234 + + // With network enabled + info := BuildInstanceInfo("abc123", "my-vm", &pid, true, 4, 4*1024*1024*1024) + assert.Equal(t, "abc123", info.ID) + assert.Equal(t, "my-vm", info.Name) + assert.Equal(t, &pid, info.HypervisorPID) + assert.Equal(t, 4, info.AllocatedVcpus) + assert.Equal(t, int64(4*1024*1024*1024), info.AllocatedMemoryBytes) + assert.NotEmpty(t, info.TAPDevice, "should have TAP device when network enabled") + + // Without network enabled + info = BuildInstanceInfo("abc123", "my-vm", &pid, false, 4, 4*1024*1024*1024) + assert.Empty(t, info.TAPDevice, "should not have TAP device when network disabled") +} + +func ptrFloat64(v float64) *float64 { + return &v +} diff --git a/lib/resources/utilization_metrics.go b/lib/vm_metrics/metrics.go similarity index 72% rename from lib/resources/utilization_metrics.go rename to lib/vm_metrics/metrics.go index 274dba3d..6bb64e90 100644 --- a/lib/resources/utilization_metrics.go +++ b/lib/vm_metrics/metrics.go @@ -1,4 +1,4 @@ -package resources +package vm_metrics import ( "context" @@ -7,8 +7,8 @@ import ( "go.opentelemetry.io/otel/metric" ) -// UtilizationMetrics holds the observable instruments for VM utilization. -type UtilizationMetrics struct { +// otelMetrics holds the OpenTelemetry instruments for VM metrics. +type otelMetrics struct { cpuSecondsTotal metric.Float64ObservableCounter allocatedVcpus metric.Int64ObservableGauge memoryRSSBytes metric.Int64ObservableGauge @@ -19,9 +19,8 @@ type UtilizationMetrics struct { memoryUtilizationRatio metric.Float64ObservableGauge } -// newUtilizationMetrics creates and registers all VM utilization metrics. -// These are observable gauges/counters that read from /proc and TAP interfaces. -func newUtilizationMetrics(meter metric.Meter, m *Manager) (*UtilizationMetrics, error) { +// newOTelMetrics creates and registers all VM utilization metrics. +func newOTelMetrics(meter metric.Meter, m *Manager) (*otelMetrics, error) { // CPU time in seconds (converted from microseconds) cpuSecondsTotal, err := meter.Float64ObservableCounter( "hypeman_vm_cpu_seconds_total", @@ -105,36 +104,34 @@ func newUtilizationMetrics(meter metric.Meter, m *Manager) (*UtilizationMetrics, // Register the callback that will collect all utilization metrics _, err = meter.RegisterCallback( func(ctx context.Context, o metric.Observer) error { - utilizations, err := m.CollectVMUtilization(ctx) + stats, err := m.CollectAll(ctx) if err != nil { // Log error but don't fail the callback return nil } - for _, util := range utilizations { + for _, s := range stats { attrs := metric.WithAttributes( - attribute.String("instance_id", util.InstanceID), - attribute.String("instance_name", util.InstanceName), + attribute.String("instance_id", s.InstanceID), + attribute.String("instance_name", s.InstanceName), ) - // Convert CPU microseconds to seconds - cpuSeconds := float64(util.CPUUsec) / 1_000_000.0 - o.ObserveFloat64(cpuSecondsTotal, cpuSeconds, attrs) + // CPU time in seconds + o.ObserveFloat64(cpuSecondsTotal, s.CPUSeconds(), attrs) // Allocated resources - o.ObserveInt64(allocatedVcpus, int64(util.AllocatedVcpus), attrs) - o.ObserveInt64(allocatedMemoryBytes, util.AllocatedMemoryBytes, attrs) + o.ObserveInt64(allocatedVcpus, int64(s.AllocatedVcpus), attrs) + o.ObserveInt64(allocatedMemoryBytes, s.AllocatedMemoryBytes, attrs) // Actual usage - o.ObserveInt64(memoryRSSBytes, int64(util.MemoryRSSBytes), attrs) - o.ObserveInt64(memoryVMSBytes, int64(util.MemoryVMSBytes), attrs) - o.ObserveInt64(networkRxBytesTotal, int64(util.NetRxBytes), attrs) - o.ObserveInt64(networkTxBytesTotal, int64(util.NetTxBytes), attrs) - - // Compute utilization ratio (RSS vs allocated) - if util.AllocatedMemoryBytes > 0 { - memRatio := float64(util.MemoryRSSBytes) / float64(util.AllocatedMemoryBytes) - o.ObserveFloat64(memoryUtilizationRatio, memRatio, attrs) + o.ObserveInt64(memoryRSSBytes, int64(s.MemoryRSSBytes), attrs) + o.ObserveInt64(memoryVMSBytes, int64(s.MemoryVMSBytes), attrs) + o.ObserveInt64(networkRxBytesTotal, int64(s.NetRxBytes), attrs) + o.ObserveInt64(networkTxBytesTotal, int64(s.NetTxBytes), attrs) + + // Compute utilization ratio + if ratio := s.MemoryUtilizationRatio(); ratio != nil { + o.ObserveFloat64(memoryUtilizationRatio, *ratio, attrs) } } @@ -153,7 +150,7 @@ func newUtilizationMetrics(meter metric.Meter, m *Manager) (*UtilizationMetrics, return nil, err } - return &UtilizationMetrics{ + return &otelMetrics{ cpuSecondsTotal: cpuSecondsTotal, allocatedVcpus: allocatedVcpus, memoryRSSBytes: memoryRSSBytes, diff --git a/lib/vm_metrics/metrics_test.go b/lib/vm_metrics/metrics_test.go new file mode 100644 index 00000000..65bee0d3 --- /dev/null +++ b/lib/vm_metrics/metrics_test.go @@ -0,0 +1,75 @@ +package vm_metrics + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestOTelMetrics_Registration(t *testing.T) { + // Create a test meter + reader := metric.NewManualReader() + provider := metric.NewMeterProvider(metric.WithReader(reader)) + meter := provider.Meter("test") + + // Create manager with mock source + mgr := NewManager() + pid := os.Getpid() + mgr.SetInstanceSource(&mockInstanceSource{ + instances: []InstanceInfo{ + { + ID: "test-vm", + Name: "Test VM", + HypervisorPID: &pid, + AllocatedVcpus: 2, + AllocatedMemoryBytes: 2 * 1024 * 1024 * 1024, + }, + }, + }) + + // Initialize OTel + err := mgr.InitializeOTel(meter) + require.NoError(t, err) + + // Collect metrics + var rm metricdata.ResourceMetrics + err = reader.Collect(t.Context(), &rm) + require.NoError(t, err) + + // Verify we have scope metrics + require.NotEmpty(t, rm.ScopeMetrics, "should have scope metrics") + + // Find our metrics + metricNames := make(map[string]bool) + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + metricNames[m.Name] = true + } + } + + // Check expected metrics are present + expectedMetrics := []string{ + "hypeman_vm_cpu_seconds_total", + "hypeman_vm_allocated_vcpus", + "hypeman_vm_memory_rss_bytes", + "hypeman_vm_memory_vms_bytes", + "hypeman_vm_allocated_memory_bytes", + "hypeman_vm_network_rx_bytes_total", + "hypeman_vm_network_tx_bytes_total", + "hypeman_vm_memory_utilization_ratio", + } + + for _, expected := range expectedMetrics { + assert.True(t, metricNames[expected], "should have metric %s", expected) + } +} + +func TestOTelMetrics_NilMeter(t *testing.T) { + mgr := NewManager() + err := mgr.InitializeOTel(nil) + require.NoError(t, err, "nil meter should not error") +} diff --git a/lib/vm_metrics/types.go b/lib/vm_metrics/types.go new file mode 100644 index 00000000..1e4b197d --- /dev/null +++ b/lib/vm_metrics/types.go @@ -0,0 +1,61 @@ +// Package vm_metrics provides real-time resource utilization metrics for VMs. +// It collects CPU, memory, and network statistics from the host's perspective +// by reading /proc//stat, /proc//statm, and TAP interface statistics. +package vm_metrics + +// VMStats holds resource utilization metrics for a single VM. +// These are point-in-time values collected from the hypervisor process. +type VMStats struct { + InstanceID string + InstanceName string + + // CPU stats (from /proc//stat) + CPUUsec uint64 // Total CPU time in microseconds (user + system) + + // Memory stats (from /proc//statm) + MemoryRSSBytes uint64 // Resident Set Size - actual physical memory used + MemoryVMSBytes uint64 // Virtual Memory Size - total allocated virtual memory + + // Network stats (from TAP interface) + NetRxBytes uint64 // Total network bytes received + NetTxBytes uint64 // Total network bytes transmitted + + // Allocated resources (for computing utilization ratios) + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes +} + +// CPUSeconds returns CPU time in seconds (for API responses). +func (s *VMStats) CPUSeconds() float64 { + return float64(s.CPUUsec) / 1_000_000.0 +} + +// MemoryUtilizationRatio returns RSS / allocated memory (0.0 to 1.0+). +// Returns nil if allocated memory is 0. +func (s *VMStats) MemoryUtilizationRatio() *float64 { + if s.AllocatedMemoryBytes <= 0 { + return nil + } + ratio := float64(s.MemoryRSSBytes) / float64(s.AllocatedMemoryBytes) + return &ratio +} + +// InstanceInfo contains the minimal info needed to collect VM metrics. +// This is provided by the instances package. +type InstanceInfo struct { + ID string + Name string + HypervisorPID *int // PID of the hypervisor process (nil if not running) + TAPDevice string // Name of the TAP device (e.g., "hype-01234567") + + // Allocated resources + AllocatedVcpus int // Number of allocated vCPUs + AllocatedMemoryBytes int64 // Allocated memory in bytes (Size + HotplugSize) +} + +// InstanceSource provides access to running instance information. +// Implemented by instances.Manager. +type InstanceSource interface { + // ListRunningInstancesForMetrics returns info for all running instances. + ListRunningInstancesForMetrics() ([]InstanceInfo, error) +} From 20dc8ee0e9aec61f9ede71ac218b0b22c0dd70e6 Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Fri, 23 Jan 2026 11:18:54 -0500 Subject: [PATCH 6/7] fix: swap TAP rx/tx for correct VM perspective TAP interface statistics are from the host's perspective: - TAP rx_bytes = bytes host receives = bytes VM transmits - TAP tx_bytes = bytes host transmits = bytes VM receives The API documents these as "bytes received/transmitted by the VM", so we need to swap them to match the VM's perspective. --- lib/vm_metrics/collector.go | 3 +++ lib/vm_metrics/manager.go | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/vm_metrics/collector.go b/lib/vm_metrics/collector.go index e9149e10..cb0343f6 100644 --- a/lib/vm_metrics/collector.go +++ b/lib/vm_metrics/collector.go @@ -93,6 +93,9 @@ func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { // ReadTAPStats reads network statistics from a TAP device. // Reads from /sys/class/net//statistics/{rx,tx}_bytes. +// Note: Returns stats from host perspective. Caller must swap for VM perspective: +// - rxBytes = host receives = VM transmits +// - txBytes = host transmits = VM receives func ReadTAPStats(tapName string) (rxBytes, txBytes uint64, err error) { basePath := filepath.Join("/sys/class/net", tapName, "statistics") diff --git a/lib/vm_metrics/manager.go b/lib/vm_metrics/manager.go index efdad42b..112047d8 100644 --- a/lib/vm_metrics/manager.go +++ b/lib/vm_metrics/manager.go @@ -89,8 +89,11 @@ func (m *Manager) GetInstanceStats(ctx context.Context, info InstanceInfo) *VMSt if err != nil { log.DebugContext(ctx, "failed to read TAP stats", "instance_id", info.ID, "tap", info.TAPDevice, "error", err) } else { - stats.NetRxBytes = rxBytes - stats.NetTxBytes = txBytes + // TAP stats are from host perspective, swap for VM perspective: + // - TAP rx_bytes = host receives = VM transmits + // - TAP tx_bytes = host transmits = VM receives + stats.NetRxBytes = txBytes + stats.NetTxBytes = rxBytes } } From 01f824026f3dd248d3ebef45423cd7e5171cd75d Mon Sep 17 00:00:00 2001 From: hiroTamada <88675973+hiroTamada@users.noreply.github.com> Date: Fri, 23 Jan 2026 13:01:53 -0500 Subject: [PATCH 7/7] docs: clarify USER_HZ vs CONFIG_HZ for clock tick rate The PR comment raised a valid concern about hardcoded ticksPerSecond=100. However, /proc always reports CPU times in USER_HZ (not kernel CONFIG_HZ): - CONFIG_HZ: kernel's internal tick rate (100, 250, 300, or 1000) - USER_HZ: userspace ABI constant, always 100 on Linux since 2.4 The kernel converts from CONFIG_HZ to USER_HZ when writing to /proc to maintain a stable userspace ABI. Added documentation explaining this. See: https://man7.org/linux/man-pages/man5/proc.5.html --- lib/vm_metrics/collector.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/vm_metrics/collector.go b/lib/vm_metrics/collector.go index cb0343f6..f9ba89b9 100644 --- a/lib/vm_metrics/collector.go +++ b/lib/vm_metrics/collector.go @@ -8,6 +8,13 @@ import ( "strings" ) +// userHZ is the clock tick rate used by /proc for CPU times. +// This is USER_HZ (not kernel CONFIG_HZ), which is always 100 on Linux. +// The kernel converts internal HZ to USER_HZ when writing to /proc to +// maintain a stable userspace ABI. This has been 100 since Linux 2.4. +// See: https://man7.org/linux/man-pages/man5/proc.5.html (search for "clock ticks") +const userHZ = 100 + // ReadProcStat reads CPU time from /proc//stat. // Returns total CPU time (user + system) in microseconds. // Fields 14 and 15 are utime and stime in clock ticks. @@ -49,10 +56,8 @@ func ReadProcStat(pid int) (uint64, error) { } // Convert clock ticks to microseconds - // Clock ticks are typically 100 per second (sysconf(_SC_CLK_TCK)) - // 1 tick = 10000 microseconds (for 100 Hz) - const ticksPerSecond = 100 - const usecPerTick = 1_000_000 / ticksPerSecond + // /proc reports CPU times in USER_HZ (always 100 on Linux) + const usecPerTick = 1_000_000 / userHZ totalUsec := (utime + stime) * usecPerTick return totalUsec, nil