Merge pull request #847 from kishankavala/CLOUDSTACK-8880

Bug-ID: CLOUDSTACK-8880: calculate free memory on host before deploying Vm.  free memory = total memory - (all vm memory)With memory over-provisioning set to 1, when mgmt server starts VMs in parallel on one host, then the memory allocated on that kvm can be larger than the actual physcial memory of the kvm host.

Fixed by checking free memory on host before starting Vm.
Added test case to check memory usage on Host.
Verified Vm deploy on Host with enough capacity and also without capacity

* pr/847:
  Bug-ID: CLOUDSTACK-8880: calculate free memory on host before deploying Vm.  free memory = total memory - (all vm memory)

Signed-off-by: Rajani Karuturi <rajani.karuturi@accelerite.com>

Author: karuturi

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -268,6 +268,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     protected int _rngRateBytes = 2048;
     private File _qemuSocketsPath;
     private final String _qemuGuestAgentSocketName = "org.qemu.guest_agent.0";
+    private long _totalMemory;
 
     private final Map <String, String> _pifs = new HashMap<String, String>();
     private final Map<String, VmStats> _vmStats = new ConcurrentHashMap<String, VmStats>();
@@ -2453,6 +2454,7 @@ private Map<String, String> getVersionStrings() {
     public StartupCommand[] initialize() {
 
         final List<Object> info = getHostInfo();
+        _totalMemory = (Long)info.get(2);
 
         final StartupRoutingCommand cmd =
                 new StartupRoutingCommand((Integer)info.get(0), (Long)info.get(1), (Long)info.get(2), (Long)info.get(4), (String)info.get(3), _hypervisorType,
@@ -3586,4 +3588,8 @@ public void restoreVMSnapshotMetadata(Domain dm, String vmName, List<Ternary<Str
             }
         }
     }
+
+    public long getTotalMemory() {
+        return _totalMemory;
+    }
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java

@@ -24,6 +24,7 @@
 
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
+import org.libvirt.Domain;
 import org.libvirt.DomainInfo.DomainState;
 import org.libvirt.LibvirtException;
 
@@ -60,6 +61,17 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
         final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
         Connect conn = null;
         try {
+
+            vm = libvirtComputingResource.createVMFromSpec(vmSpec);
+            conn = libvirtUtilitiesHelper.getConnectionByType(vm.getHvsType());
+
+            Long remainingMem = getFreeMemory(conn, libvirtComputingResource);
+            if (remainingMem == null){
+                return new StartAnswer(command, "failed to get free memory");
+            } else if (remainingMem < vmSpec.getMinRam()) {
+                return new StartAnswer(command, "Not enough memory on the host, remaining: " + remainingMem + ", asking: " + vmSpec.getMinRam());
+            }
+
             final NicTO[] nics = vmSpec.getNics();
 
             for (final NicTO nic : nics) {
@@ -68,8 +80,6 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
                 }
             }
 
-            vm = libvirtComputingResource.createVMFromSpec(vmSpec);
-            conn = libvirtUtilitiesHelper.getConnectionByType(vm.getHvsType());
             libvirtComputingResource.createVbd(conn, vmSpec, vmName, vm);
 
             if (!storagePoolMgr.connectPhysicalDisksViaVmSpec(vmSpec)) {
@@ -150,4 +160,22 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
             }
         }
     }
-}
+
+    private Long getFreeMemory(final Connect conn, final LibvirtComputingResource libvirtComputingResource){
+        try {
+            long allocatedMem = 0;
+            int[] ids = conn.listDomains();
+            for(int id :ids) {
+                Domain dm = conn.domainLookupByID(id);
+                allocatedMem += dm.getMaxMemory() * 1024L;
+                s_logger.debug("vm: " + dm.getName() + " mem: " + dm.getMaxMemory() * 1024L);
+            }
+            Long remainingMem = libvirtComputingResource.getTotalMemory() - allocatedMem;
+            s_logger.debug("remaining mem" + remainingMem);
+            return remainingMem;
+        } catch (Exception e) {
+            s_logger.debug("failed to get free memory", e);
+            return null;
+        }
+    }
+}

plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java

@@ -4855,6 +4855,7 @@ public void testStartCommand() {
 
         final NicTO nic = Mockito.mock(NicTO.class);
         final NicTO[] nics = new NicTO[]{nic};
+        final int[] vms = new int[0];
 
         final String vmName = "Test";
         final String controlIp = "127.0.0.1";
@@ -4868,6 +4869,7 @@ public void testStartCommand() {
         when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
         try {
             when(libvirtUtilitiesHelper.getConnectionByType(vmDef.getHvsType())).thenReturn(conn);
+            when(conn.listDomains()).thenReturn(vms);
             doNothing().when(libvirtComputingResource).createVbd(conn, vmSpec, vmName, vmDef);
         } catch (final LibvirtException e) {
             fail(e.getMessage());
@@ -4927,6 +4929,7 @@ public void testStartCommandIsolationEc2() {
 
         final NicTO nic = Mockito.mock(NicTO.class);
         final NicTO[] nics = new NicTO[]{nic};
+        final int[] vms = new int[0];
 
         final String vmName = "Test";
         final String controlIp = "127.0.0.1";
@@ -4940,6 +4943,7 @@ public void testStartCommandIsolationEc2() {
         when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
         try {
             when(libvirtUtilitiesHelper.getConnectionByType(vmDef.getHvsType())).thenReturn(conn);
+            when(conn.listDomains()).thenReturn(vms);
             doNothing().when(libvirtComputingResource).createVbd(conn, vmSpec, vmName, vmDef);
         } catch (final LibvirtException e) {
             fail(e.getMessage());
@@ -4989,6 +4993,61 @@ public void testStartCommandIsolationEc2() {
         }
     }
 
+    @Test
+    public void testStartCommandHostMemory() {
+        final VirtualMachineTO vmSpec = Mockito.mock(VirtualMachineTO.class);
+        final com.cloud.host.Host host = Mockito.mock(com.cloud.host.Host.class);
+        final boolean executeInSequence = false;
+
+        final StartCommand command = new StartCommand(vmSpec, host, executeInSequence);
+
+        final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class);
+        final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
+        final Connect conn = Mockito.mock(Connect.class);
+        final LibvirtVMDef vmDef = Mockito.mock(LibvirtVMDef.class);
+
+        final NicTO nic = Mockito.mock(NicTO.class);
+        final NicTO[] nics = new NicTO[]{nic};
+        int vmId = 1;
+        final int[] vms = new int[]{vmId};
+        final Domain dm = Mockito.mock(Domain.class);
+
+        final String vmName = "Test";
+
+        when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr);
+        when(vmSpec.getNics()).thenReturn(nics);
+        when(vmSpec.getType()).thenReturn(VirtualMachine.Type.User);
+        when(vmSpec.getName()).thenReturn(vmName);
+        when(vmSpec.getMaxRam()).thenReturn(512L);
+        when(libvirtComputingResource.createVMFromSpec(vmSpec)).thenReturn(vmDef);
+
+        when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
+        try {
+            when(libvirtUtilitiesHelper.getConnectionByType(vmDef.getHvsType())).thenReturn(conn);
+            when(conn.listDomains()).thenReturn(vms);
+            when(conn.domainLookupByID(vmId)).thenReturn(dm);
+            when(dm.getMaxMemory()).thenReturn(1024L);
+            when(dm.getName()).thenReturn(vmName);
+            when(libvirtComputingResource.getTotalMemory()).thenReturn(2048*1024L);
+            doNothing().when(libvirtComputingResource).createVbd(conn, vmSpec, vmName, vmDef);
+        } catch (final LibvirtException e) {
+            fail(e.getMessage());
+        } catch (final InternalErrorException e) {
+            fail(e.getMessage());
+        } catch (final URISyntaxException e) {
+            fail(e.getMessage());
+        }
+
+        when(storagePoolMgr.connectPhysicalDisksViaVmSpec(vmSpec)).thenReturn(true);
+
+        final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
+        assertNotNull(wrapper);
+
+        final Answer answer = wrapper.execute(command, libvirtComputingResource);
+        assertTrue(answer.getResult());
+    }
+
+
     @Test
     public void testUpdateHostPasswordCommand() {
         final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);