Synchronization of network devices on newly added hosts for Persistent Networks (#5977)

* Persistent Network feature & Marvin component tests

* Cleaned up comments and imports

* fixed small error

* add support to add setup persistent networks' resources when a disabled host is enabled

* small fix

* use wildcard instead of hard-coding the bridge name

* allow clean up of resources when removing a host in maintenance mode

* skip test for simulator hypervisor

Co-authored-by: shatoboar <sang-woo.bae@campus.tu-berlin.de>

Author: Pearl1594

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HypervisorHostListener.java

@@ -30,4 +30,6 @@ public interface HypervisorHostListener {
     boolean hostAboutToBeRemoved(long hostId);
 
     boolean hostRemoved(long hostId, long clusterId);
+
+    boolean hostEnabled(long hostId);
 }

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -280,6 +280,8 @@ public interface StorageManager extends StorageService {
 
     void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException;
 
+    void enableHost(long hostId) throws StorageUnavailableException, StorageConflictException;
+
     void createCapacityEntry(long poolId);
 
     DataStore createLocalStorage(Host host, StoragePoolInfo poolInfo) throws ConnectionException;

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java

@@ -31,6 +31,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import org.apache.cloudstack.agent.lb.SetupMSListCommand;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.log4j.Logger;
@@ -118,7 +119,8 @@ public int compare(final Object o1, final Object o2) {
         StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(),
         ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(),
         CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(),
-        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString()};
+        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(),
+            CleanupPersistentNetworkResourceCommand.class.toString()};
     protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() };
     static {
         Arrays.sort(s_commandsAllowedInMaintenanceMode);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -130,4 +130,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType);
 
     List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri);
+
+    List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId);
 }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java

@@ -419,6 +419,16 @@ public int getOtherPersistentNetworksCount(long id, String broadcastURI, boolean
         return persistentNetworks.size();
     }
 
+    @Override
+    public List<NetworkVO> getAllPersistentNetworksFromZone(long dataCenterId) {
+        Object[] guestTypes = {"Isolated", "L2"};
+        final SearchCriteria<NetworkVO> sc = PersistentNetworkSearch.create();
+        sc.setParameters("guestType", guestTypes);
+        sc.setParameters("dc", dataCenterId);
+        sc.setJoinParameters("persistent", "persistent", true);
+        return search(sc, null);
+    }
+
     @Override
     public String getNextAvailableMacAddress(final long networkConfigId, Integer zoneMacIdentifier) {
         final SequenceFetcher fetch = SequenceFetcher.getInstance();

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -20,10 +20,21 @@
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CleanupPersistentNetworkResourceCommand;
 import com.cloud.agent.api.ModifyStoragePoolAnswer;
 import com.cloud.agent.api.ModifyStoragePoolCommand;
+import com.cloud.agent.api.SetupPersistentNetworkCommand;
+import com.cloud.agent.api.to.NicTO;
 import com.cloud.alert.AlertManager;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.exception.StorageConflictException;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.offerings.NetworkOfferingVO;
+import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
@@ -62,12 +73,50 @@ public class DefaultHostListener implements HypervisorHostListener {
     StorageManager storageManager;
     @Inject
     StorageService storageService;
+    @Inject
+    NetworkOfferingDao networkOfferingDao;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    NetworkModel networkModel;
+    @Inject
+    ConfigurationManager configManager;
+    @Inject
+    NetworkDao networkDao;
+
 
     @Override
     public boolean hostAdded(long hostId) {
         return true;
     }
 
+    private boolean createPersistentNetworkResourcesOnHost(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %ld can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    /**
+     * Creates a dummy NicTO object which is used by the respective hypervisors to setup network elements / resources
+     * - bridges(KVM), VLANs(Xen) and portgroups(VMWare) for L2 network
+     */
+    private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOfferingVO networkOfferingVO, HostVO hostVO) {
+        NicTO to = new NicTO();
+        to.setName(networkModel.getNetworkTag(hostVO.getHypervisorType(), networkVO));
+        to.setBroadcastType(networkVO.getBroadcastDomainType());
+        to.setType(networkVO.getTrafficType());
+        to.setBroadcastUri(networkVO.getBroadcastUri());
+        to.setIsolationuri(networkVO.getBroadcastUri());
+        to.setNetworkRateMbps(configManager.getNetworkOfferingNetworkRate(networkOfferingVO.getId(), networkVO.getDataCenterId()));
+        to.setSecurityGroupEnabled(networkModel.isSecurityGroupSupportedInNetwork(networkVO));
+        return to;
+    }
+
+
     @Override
     public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
         StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
@@ -109,7 +158,8 @@ public boolean hostConnect(long hostId, long poolId) throws StorageConflictExcep
         storageService.updateStorageCapabilities(poolId, false);
 
         s_logger.info("Connection established between storage pool " + pool + " and host " + hostId);
-        return true;
+
+        return createPersistentNetworkResourcesOnHost(hostId);
     }
 
     private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) {
@@ -124,7 +174,7 @@ private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, Mo
         StoragePoolVO poolVO = this.primaryStoreDao.findById(pool.getId());
         poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes());
         poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes());
-        if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) {
+        if (StringUtils.isNotEmpty(mspAnswer.getPoolType())) {
             StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(pool.getId(), "pool_type");
             if (poolType == null) {
                 StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(pool.getId(), "pool_type", mspAnswer.getPoolType(), false);
@@ -142,11 +192,62 @@ public boolean hostDisconnected(long hostId, long poolId) {
 
     @Override
     public boolean hostAboutToBeRemoved(long hostId) {
+        // send host the cleanup persistent network resources
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn("Host with id " + hostId + " can't be found");
+            return false;
+        }
+
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId()); // find zoneId of host
+        for (NetworkVO persistentNetworkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(persistentNetworkVO.getNetworkOfferingId());
+            CleanupPersistentNetworkResourceCommand cleanupCmd =
+                    new CleanupPersistentNetworkResourceCommand(createNicTOFromNetworkAndOffering(persistentNetworkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(hostId, cleanupCmd);
+            if (answer == null) {
+                s_logger.error("Unable to get answer to the cleanup persistent network command " + persistentNetworkVO.getId());
+                continue;
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to cleanup persistent network resources from network %d on the host %d", persistentNetworkVO.getId(), hostId);
+                s_logger.error(msg);
+            }
+        }
         return true;
     }
 
     @Override
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        if (host == null) {
+            s_logger.warn(String.format("Host with id %d can't be found", hostId));
+            return false;
+        }
+        setupPersistentNetwork(host);
+        return true;
+    }
+
+    private void setupPersistentNetwork(HostVO host) {
+        List<NetworkVO> allPersistentNetworks = networkDao.getAllPersistentNetworksFromZone(host.getDataCenterId());
+        for (NetworkVO networkVO : allPersistentNetworks) {
+            NetworkOfferingVO networkOfferingVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId());
+
+            SetupPersistentNetworkCommand persistentNetworkCommand =
+                    new SetupPersistentNetworkCommand(createNicTOFromNetworkAndOffering(networkVO, networkOfferingVO, host));
+            Answer answer = agentMgr.easySend(host.getId(), persistentNetworkCommand);
+            if (answer == null) {
+                throw new CloudRuntimeException("Unable to get answer to the setup persistent network command " + networkVO.getId());
+            }
+            if (!answer.getResult()) {
+                String msg = String.format("Unable to create persistent network resources for network %d on the host %d in zone %d", networkVO.getId(), host.getId(), networkVO.getDataCenterId());
+                s_logger.error(msg);
+            }
+        }
+    }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java

@@ -140,4 +140,9 @@ public boolean hostAboutToBeRemoved(long hostId) {
     public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }

plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/provider/ElastistorHostListener.java

@@ -178,6 +178,11 @@ public boolean hostRemoved(long hostId, long clusterId) {
         return true;
     }
 
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
+
     private void handleXenServer(long clusterId, long hostId, long storagePoolId) {
         List<String> storagePaths = getStoragePaths(clusterId, storagePoolId);
 

plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java

@@ -59,4 +59,9 @@ public boolean hostRemoved(long hostId, long clusterId) {
 
         return true;
     }
+
+    @Override
+    public boolean hostEnabled(long hostId) {
+        return true;
+    }
 }