feat(metrics): add per-VM resource utilization metrics

cmd/api/api/api.go

@@ -10,20 +10,22 @@ import (
 	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/resources"
+	"github.com/kernel/hypeman/lib/vm_metrics"
 	"github.com/kernel/hypeman/lib/volumes"
 )
 
 // ApiService implements the oapi.StrictServerInterface
 type ApiService struct {
-	Config          *config.Config
-	ImageManager    images.Manager
-	InstanceManager instances.Manager
-	VolumeManager   volumes.Manager
-	NetworkManager  network.Manager
-	DeviceManager   devices.Manager
-	IngressManager  ingress.Manager
-	BuildManager    builds.Manager
-	ResourceManager *resources.Manager
+	Config           *config.Config
+	ImageManager     images.Manager
+	InstanceManager  instances.Manager
+	VolumeManager    volumes.Manager
+	NetworkManager   network.Manager
+	DeviceManager    devices.Manager
+	IngressManager   ingress.Manager
+	BuildManager     builds.Manager
+	ResourceManager  *resources.Manager
+	VMMetricsManager *vm_metrics.Manager
 }
 
 var _ oapi.StrictServerInterface = (*ApiService)(nil)
@@ -39,16 +41,18 @@ func New(
 	ingressManager ingress.Manager,
 	buildManager builds.Manager,
 	resourceManager *resources.Manager,
+	vmMetricsManager *vm_metrics.Manager,
 ) *ApiService {
 	return &ApiService{
-		Config:          config,
-		ImageManager:    imageManager,
-		InstanceManager: instanceManager,
-		VolumeManager:   volumeManager,
-		NetworkManager:  networkManager,
-		DeviceManager:   deviceManager,
-		IngressManager:  ingressManager,
-		BuildManager:    buildManager,
-		ResourceManager: resourceManager,
+		Config:           config,
+		ImageManager:     imageManager,
+		InstanceManager:  instanceManager,
+		VolumeManager:    volumeManager,
+		NetworkManager:   networkManager,
+		DeviceManager:    deviceManager,
+		IngressManager:   ingressManager,
+		BuildManager:     buildManager,
+		ResourceManager:  resourceManager,
+		VMMetricsManager: vmMetricsManager,
 	}
 }

cmd/api/api/instances.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/resources"
+	"github.com/kernel/hypeman/lib/vm_metrics"
 	"github.com/samber/lo"
 )
 
@@ -274,6 +275,52 @@ func (s *ApiService) GetInstance(ctx context.Context, request oapi.GetInstanceRe
 	return oapi.GetInstance200JSONResponse(instanceToOAPI(*inst)), nil
 }
 
+// GetInstanceStats returns resource utilization statistics for an instance
+// The id parameter can be an instance ID, name, or ID prefix
+// Note: Resolution is handled by ResolveResource middleware
+func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInstanceStatsRequestObject) (oapi.GetInstanceStatsResponseObject, error) {
+	inst := mw.GetResolvedInstance[instances.Instance](ctx)
+	if inst == nil {
+		return oapi.GetInstanceStats500JSONResponse{
+			Code:    "internal_error",
+			Message: "resource not resolved",
+		}, nil
+	}
+
+	// Build instance info for metrics collection
+	info := vm_metrics.BuildInstanceInfo(
+		inst.Id,
+		inst.Name,
+		inst.HypervisorPID,
+		inst.NetworkEnabled,
+		inst.Vcpus,
+		inst.Size+inst.HotplugSize,
+	)
+
+	// Collect stats using vm_metrics manager
+	vmStats := s.VMMetricsManager.GetInstanceStats(ctx, info)
+
+	// Map domain type to API type
+	return oapi.GetInstanceStats200JSONResponse(vmStatsToOAPI(vmStats)), nil
+}
+
+// vmStatsToOAPI converts vm_metrics.VMStats to oapi.InstanceStats
+func vmStatsToOAPI(s *vm_metrics.VMStats) oapi.InstanceStats {
+	stats := oapi.InstanceStats{
+		InstanceId:           s.InstanceID,
+		InstanceName:         s.InstanceName,
+		CpuSeconds:           s.CPUSeconds(),
+		MemoryRssBytes:       int64(s.MemoryRSSBytes),
+		MemoryVmsBytes:       int64(s.MemoryVMSBytes),
+		NetworkRxBytes:       int64(s.NetRxBytes),
+		NetworkTxBytes:       int64(s.NetTxBytes),
+		AllocatedVcpus:       s.AllocatedVcpus,
+		AllocatedMemoryBytes: s.AllocatedMemoryBytes,
+		MemoryUtilizationRatio: s.MemoryUtilizationRatio(),
+	}
+	return stats
+}
+
 // DeleteInstance stops and deletes an instance
 // The id parameter can be an instance ID, name, or ID prefix
 // Note: Resolution is handled by ResolveResource middleware

cmd/api/wire.go

@@ -19,25 +19,27 @@ import (
 	"github.com/kernel/hypeman/lib/registry"
 	"github.com/kernel/hypeman/lib/resources"
 	"github.com/kernel/hypeman/lib/system"
+	"github.com/kernel/hypeman/lib/vm_metrics"
 	"github.com/kernel/hypeman/lib/volumes"
 )
 
 // application struct to hold initialized components
 type application struct {
-	Ctx             context.Context
-	Logger          *slog.Logger
-	Config          *config.Config
-	ImageManager    images.Manager
-	SystemManager   system.Manager
-	NetworkManager  network.Manager
-	DeviceManager   devices.Manager
-	InstanceManager instances.Manager
-	VolumeManager   volumes.Manager
-	IngressManager  ingress.Manager
-	BuildManager    builds.Manager
-	ResourceManager *resources.Manager
-	Registry        *registry.Registry
-	ApiService      *api.ApiService
+	Ctx              context.Context
+	Logger           *slog.Logger
+	Config           *config.Config
+	ImageManager     images.Manager
+	SystemManager    system.Manager
+	NetworkManager   network.Manager
+	DeviceManager    devices.Manager
+	InstanceManager  instances.Manager
+	VolumeManager    volumes.Manager
+	IngressManager   ingress.Manager
+	BuildManager     builds.Manager
+	ResourceManager  *resources.Manager
+	VMMetricsManager *vm_metrics.Manager
+	Registry         *registry.Registry
+	ApiService       *api.ApiService
 }
 
 // initializeApp is the injector function
@@ -56,6 +58,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideIngressManager,
 		providers.ProvideBuildManager,
 		providers.ProvideResourceManager,
+		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,
 		wire.Struct(new(application), "*"),

dashboards/hypeman.json

@@ -417,6 +417,112 @@
       ],
       "title": "Exec Sessions & Duration",
       "type": "timeseries"
+    },
+    {
+      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 },
+      "id": 18,
+      "title": "VM Resource Utilization",
+      "type": "row"
+    },
+    {
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 },
+      "id": 19,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(hypeman_vm_cpu_seconds_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])",
+          "legendFormat": "{{instance_name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VM CPU Usage (cores)",
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": { "unit": "short", "min": 0 },
+        "overrides": []
+      }
+    },
+    {
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 },
+      "id": 20,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "hypeman_vm_memory_rss_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}",
+          "legendFormat": "{{instance_name}} RSS",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "hypeman_vm_memory_vms_bytes{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}",
+          "legendFormat": "{{instance_name}} VMS",
+          "refId": "B"
+        }
+      ],
+      "title": "VM Memory Usage (RSS & VMS)",
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": { "unit": "bytes", "min": 0 },
+        "overrides": []
+      }
+    },
+    {
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 },
+      "id": 21,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(hypeman_vm_network_rx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])",
+          "legendFormat": "{{instance_name}} RX",
+          "refId": "A"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "rate(hypeman_vm_network_tx_bytes_total{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}[1m])",
+          "legendFormat": "{{instance_name}} TX",
+          "refId": "B"
+        }
+      ],
+      "title": "VM Network I/O",
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": { "unit": "Bps", "min": 0 },
+        "overrides": []
+      }
+    },
+    {
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 },
+      "id": 23,
+      "options": {
+        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "hypeman_vm_memory_utilization_ratio{deployment_environment_name=~\"$env\", service_instance_id=~\"$instance\"}",
+          "legendFormat": "{{instance_name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "VM Memory Utilization (% of allocated)",
+      "type": "timeseries",
+      "fieldConfig": {
+        "defaults": { "unit": "percentunit", "min": 0, "max": 1 },
+        "overrides": []
+      }
     }
   ],
   "refresh": "10s",

lib/builds/manager_test.go

@@ -122,6 +122,10 @@ func (m *mockInstanceManager) ListInstanceAllocations(ctx context.Context) ([]re
 	return nil, nil
 }
 
+func (m *mockInstanceManager) ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) {
+	return nil, nil
+}
+
 // mockVolumeManager implements volumes.Manager for testing
 type mockVolumeManager struct {
 	volumes               map[string]*volumes.Volume