Skip to content

Commit 32e3bbd

Browse files
harikrishna-patnalaharikrishna-patnala
authored
VMware Datastore Cluster primary storage pool synchronisation (#4871)
Datastore cluster as a primary storage support is already there. But if any changes at vCenter to datastore cluster like addition/removal of datastore is not synchronised with CloudStack directly. It needs removal of primary storage from CloudStack and add it again to CloudStack. Here synchronisation of datastore cluster is fixed without need to remove or add the datastore cluster. 1. A new API is introduced syncStoragePool which takes datastore cluster storage pool UUID as the parameter. This API checks if there any changes in the datastore cluster and updates management server accordingly. 2. During synchronisation if a new child datastore is found in datastore cluster, then management server will create a new child storage pool in database under the datastore cluster. If the new child storage pool is already added as an individual storage pool then the existing storage pool entry will be converted to child storage pool (instead of creating a new storage pool entry) 3. During synchronisaton if the existing child datastore in CloudStack is found to be removed on vCenter then management server removes that child datastore from datastore cluster and makes it an individual storage pool. The above behaviour is on par with the vCenter behaviour when adding and removing child datastore.
1 parent 1eea9c5 commit 32e3bbd

File tree

24 files changed

+658
-57
lines changed

24 files changed

+658
-57
lines changed

‎api/src/main/java/com/cloud/event/EventTypes.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,7 @@ public class EventTypes {
374374
// Primary storage pool
375375
public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS";
376376
public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS";
377+
public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL";
377378

378379
// VPN
379380
public static final String EVENT_REMOTE_ACCESS_VPN_CREATE = "VPN.REMOTE.ACCESS.CREATE";

‎api/src/main/java/com/cloud/storage/StorageService.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
2828
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
2929
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
30+
import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd;
3031

3132
import com.cloud.exception.DiscoveryException;
3233
import com.cloud.exception.InsufficientCapacityException;
@@ -104,4 +105,6 @@ public interface StorageService {
104105

105106
ImageStore updateImageStoreStatus(Long id, Boolean readonly);
106107

108+
StoragePool syncStoragePool(SyncStoragePoolCmd cmd);
109+
107110
}

‎api/src/main/java/org/apache/cloudstack/api/command/admin/storage/SyncStoragePoolCmd.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.api.command.admin.storage;
18+
19+
import com.cloud.event.EventTypes;
20+
import com.cloud.exception.ResourceUnavailableException;
21+
import com.cloud.exception.InsufficientCapacityException;
22+
import com.cloud.exception.ConcurrentOperationException;
23+
import com.cloud.exception.ResourceAllocationException;
24+
import com.cloud.exception.NetworkRuleConflictException;
25+
import com.cloud.storage.StoragePool;
26+
import org.apache.cloudstack.acl.RoleType;
27+
import org.apache.cloudstack.api.APICommand;
28+
import org.apache.cloudstack.api.ApiConstants;
29+
import org.apache.cloudstack.api.BaseAsyncCmd;
30+
import org.apache.cloudstack.api.Parameter;
31+
import org.apache.cloudstack.api.ServerApiException;
32+
import org.apache.cloudstack.api.response.StoragePoolResponse;
33+
import org.apache.cloudstack.api.ApiErrorCode;
34+
import org.apache.cloudstack.context.CallContext;
35+
36+
import java.util.logging.Logger;
37+
38+
@APICommand(name = SyncStoragePoolCmd.APINAME,
39+
description = "Sync storage pool with management server (currently supported for Datastore Cluster in VMware and syncs the datastores in it)",
40+
responseObject = StoragePoolResponse.class,
41+
requestHasSensitiveInfo = false,
42+
responseHasSensitiveInfo = false,
43+
since = "4.15.1",
44+
authorized = {RoleType.Admin}
45+
)
46+
public class SyncStoragePoolCmd extends BaseAsyncCmd {
47+
48+
public static final String APINAME = "syncStoragePool";
49+
public static final Logger LOGGER = Logger.getLogger(SyncStoragePoolCmd.class.getName());
50+
51+
/////////////////////////////////////////////////////
52+
//////////////// API parameters /////////////////////
53+
/////////////////////////////////////////////////////
54+
55+
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, required = true, description = "Storage pool id")
56+
private Long poolId;
57+
58+
/////////////////////////////////////////////////////
59+
/////////////////// Accessors ///////////////////////
60+
/////////////////////////////////////////////////////
61+
62+
public Long getPoolId() {
63+
return poolId;
64+
}
65+
66+
@Override
67+
public String getEventType() {
68+
return EventTypes.EVENT_SYNC_STORAGE_POOL;
69+
}
70+
71+
@Override
72+
public String getEventDescription() {
73+
return "Attempting to synchronise storage pool with management server";
74+
}
75+
76+
@Override
77+
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
78+
StoragePool result = _storageService.syncStoragePool(this);
79+
if (result != null) {
80+
StoragePoolResponse response = _responseGenerator.createStoragePoolResponse(result);
81+
response.setResponseName("storagepool");
82+
this.setResponseObject(response);
83+
} else {
84+
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to synchronise storage pool");
85+
}
86+
}
87+
88+
@Override
89+
public String getCommandName() {
90+
return APINAME.toLowerCase() + BaseAsyncCmd.RESPONSE_SUFFIX;
91+
}
92+
93+
@Override
94+
public long getEntityOwnerId() {
95+
return CallContext.current().getCallingAccountId();
96+
}
97+
}

‎core/src/main/java/com/cloud/storage/resource/StorageProcessor.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.cloudstack.storage.command.IntroduceObjectCmd;
3131
import org.apache.cloudstack.storage.command.ResignatureCommand;
3232
import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand;
33+
import org.apache.cloudstack.storage.command.SyncVolumePathCommand;
3334

3435
import com.cloud.agent.api.Answer;
3536

@@ -81,5 +82,7 @@ public interface StorageProcessor {
8182

8283
Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd);
8384

84-
public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd);
85+
public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd);
86+
87+
public Answer syncVolumePath(SyncVolumePathCommand cmd);
8588
}

‎core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.cloudstack.storage.command.ResignatureCommand;
3535
import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand;
3636
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
37+
import org.apache.cloudstack.storage.command.SyncVolumePathCommand;
3738

3839
import com.cloud.agent.api.Answer;
3940
import com.cloud.agent.api.Command;
@@ -73,7 +74,9 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) {
7374
} else if (command instanceof DirectDownloadCommand) {
7475
return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command);
7576
} else if (command instanceof CheckDataStoreStoragePolicyComplainceCommand) {
76-
return processor.CheckDataStoreStoragePolicyComplaince((CheckDataStoreStoragePolicyComplainceCommand) command);
77+
return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplainceCommand) command);
78+
} else if (command instanceof SyncVolumePathCommand) {
79+
return processor.syncVolumePath((SyncVolumePathCommand) command);
7780
}
7881

7982
return new Answer((Command)command, false, "not implemented yet");

‎core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathAnswer.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package org.apache.cloudstack.storage.command;
21+
22+
import com.cloud.agent.api.Answer;
23+
import com.cloud.agent.api.to.DiskTO;
24+
25+
public class SyncVolumePathAnswer extends Answer {
26+
private DiskTO disk;
27+
28+
public SyncVolumePathAnswer() {
29+
super(null);
30+
}
31+
32+
public SyncVolumePathAnswer(DiskTO disk) {
33+
super(null);
34+
setDisk(disk);
35+
}
36+
37+
public SyncVolumePathAnswer(String errMsg) {
38+
super(null, false, errMsg);
39+
}
40+
41+
public DiskTO getDisk() {
42+
return disk;
43+
}
44+
45+
public void setDisk(DiskTO disk) {
46+
this.disk = disk;
47+
}
48+
49+
}

‎core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathCommand.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
package org.apache.cloudstack.storage.command;
20+
21+
import com.cloud.agent.api.to.DiskTO;
22+
23+
public class SyncVolumePathCommand extends StorageSubSystemCommand {
24+
25+
private DiskTO disk;
26+
27+
public SyncVolumePathCommand(final DiskTO disk) {
28+
super();
29+
this.disk = disk;
30+
}
31+
32+
public DiskTO getDisk() {
33+
return disk;
34+
}
35+
36+
public void setDisk(final DiskTO disk) {
37+
this.disk = disk;
38+
}
39+
40+
@Override
41+
public boolean executeInSequence() {
42+
return false;
43+
}
44+
45+
@Override
46+
public void setExecuteInSequence(boolean inSeq) {
47+
48+
}
49+
}

‎engine/components-api/src/main/java/com/cloud/storage/StorageManager.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.math.BigDecimal;
2020
import java.util.List;
2121

22+
import com.cloud.agent.api.ModifyStoragePoolAnswer;
2223
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2324
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
2425
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -240,4 +241,6 @@ public interface StorageManager extends StorageService {
240241

241242
boolean isStoragePoolDatastoreClusterParent(StoragePool pool);
242243

244+
void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List<ModifyStoragePoolAnswer> childDatastoreAnswerList, long hostId);
245+
243246
}

‎engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ public interface StoragePoolHostDao extends GenericDao<StoragePoolHostVO, Long>
3232

3333
List<StoragePoolHostVO> listByHostStatus(long poolId, Status hostStatus);
3434

35+
List<Long> findHostsConnectedToPools(List<Long> poolIds);
36+
3537
List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
3638

3739
public void deletePrimaryRecordsForHost(long hostId);

‎engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import java.sql.SQLException;
2222
import java.util.ArrayList;
2323
import java.util.List;
24-
24+
import java.util.stream.Collectors;
2525

2626
import org.apache.log4j.Logger;
2727
import org.springframework.stereotype.Component;
@@ -44,6 +44,8 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
4444

4545
protected static final String HOST_FOR_POOL_SEARCH = "SELECT * FROM storage_pool_host_ref ph, host h where ph.host_id = h.id and ph.pool_id=? and h.status=? ";
4646

47+
protected static final String HOSTS_FOR_POOLS_SEARCH = "SELECT DISTINCT(ph.host_id) FROM storage_pool_host_ref ph, host h WHERE ph.host_id = h.id AND h.status = 'Up' AND resource_state = 'Enabled' AND ph.pool_id IN (?)";
48+
4749
protected static final String STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id, count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
4850
+ " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
4951

@@ -121,6 +123,33 @@ public List<StoragePoolHostVO> listByHostStatus(long poolId, Status hostStatus)
121123
return result;
122124
}
123125

126+
@Override
127+
public List<Long> findHostsConnectedToPools(List<Long> poolIds) {
128+
List<Long> hosts = new ArrayList<Long>();
129+
if (poolIds == null || poolIds.isEmpty()) {
130+
return hosts;
131+
}
132+
133+
String poolIdsInStr = poolIds.stream().map(poolId -> String.valueOf(poolId)).collect(Collectors.joining(",", "(", ")"));
134+
String sql = HOSTS_FOR_POOLS_SEARCH.replace("(?)", poolIdsInStr);
135+
136+
TransactionLegacy txn = TransactionLegacy.currentTxn();
137+
try(PreparedStatement pstmt = txn.prepareStatement(sql);) {
138+
try(ResultSet rs = pstmt.executeQuery();) {
139+
while (rs.next()) {
140+
long hostId = rs.getLong(1); // host_id column
141+
hosts.add(hostId);
142+
}
143+
} catch (SQLException e) {
144+
s_logger.warn(String.format("Unable to retrieve hosts from pools [%s] due to [%s].", poolIdsInStr, e.getMessage()));
145+
}
146+
} catch (Exception e) {
147+
s_logger.warn(String.format("Unable to retrieve hosts from pools [%s] due to [%s].", poolIdsInStr, e.getMessage()));
148+
}
149+
150+
return hosts;
151+
}
152+
124153
@Override
125154
public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
126155
ArrayList<Pair<Long, Integer>> l = new ArrayList<Pair<Long, Integer>>();

0 commit comments

Comments
 (0)