Skip to content

Commit 9e03f4b

Browse files
Pearl1594Pearl1594
committed
CLVM enhancements and fixes
1 parent d3e1976 commit 9e03f4b

File tree

3 files changed

+269
-16
lines changed

3 files changed

+269
-16
lines changed

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 96 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,7 +1115,14 @@ public Answer backupSnapshot(final CopyCommand cmd) {
11151115
}
11161116
} else {
11171117
final Script command = new Script(_manageSnapshotPath, cmd.getWaitInMillSeconds(), logger);
1118-
command.add("-b", isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath());
1118+
String backupPath;
1119+
if (primaryPool.getType() == StoragePoolType.CLVM) {
1120+
backupPath = snapshotDisk.getPath();
1121+
logger.debug("Using snapshotDisk path for CLVM backup: " + backupPath);
1122+
} else {
1123+
backupPath = isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath();
1124+
}
1125+
command.add("-b", backupPath);
11191126
command.add(NAME_OPTION, snapshotName);
11201127
command.add("-p", snapshotDestPath);
11211128

@@ -1172,7 +1179,11 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject
11721179

11731180
if ((backupSnapshotAfterTakingSnapshot == null || BooleanUtils.toBoolean(backupSnapshotAfterTakingSnapshot)) && deleteSnapshotOnPrimary) {
11741181
try {
1175-
Files.deleteIfExists(Paths.get(snapshotPath));
1182+
if (primaryPool.getType() == StoragePoolType.CLVM) {
1183+
deleteClvmSnapshot(snapshotPath);
1184+
} else {
1185+
Files.deleteIfExists(Paths.get(snapshotPath));
1186+
}
11761187
} catch (IOException ex) {
11771188
logger.error("Failed to delete snapshot [{}] on primary storage [{}].", snapshot.getId(), snapshot.getName(), ex);
11781189
}
@@ -1181,6 +1192,81 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject
11811192
}
11821193
}
11831194

1195+
/**
1196+
* Delete a CLVM snapshot using lvremove command.
1197+
* For CLVM, the snapshot path stored in DB is: /dev/vgname/volumeuuid/snapshotuuid
1198+
* However, managesnapshot.sh creates the actual snapshot using MD5 hash of the snapshot UUID.
1199+
* The actual device is at: /dev/mapper/vgname-MD5(snapshotuuid)
1200+
* We need to compute the MD5 hash and remove both the snapshot LV and its COW volume.
1201+
*/
1202+
private void deleteClvmSnapshot(String snapshotPath) {
1203+
try {
1204+
// Parse the snapshot path: /dev/acsvg/volume-uuid/snapshot-uuid
1205+
// Extract VG name and snapshot UUID
1206+
String[] pathParts = snapshotPath.split("/");
1207+
if (pathParts.length < 5) {
1208+
logger.warn("Invalid CLVM snapshot path format: " + snapshotPath + ", skipping deletion");
1209+
return;
1210+
}
1211+
1212+
String vgName = pathParts[2];
1213+
String snapshotUuid = pathParts[4];
1214+
1215+
// Compute MD5 hash of snapshot UUID (same as managesnapshot.sh does)
1216+
String md5Hash = computeMd5Hash(snapshotUuid);
1217+
1218+
logger.debug("Deleting CLVM snapshot for UUID: " + snapshotUuid + " (MD5: " + md5Hash + ")");
1219+
1220+
// Remove the snapshot device mapper entry
1221+
// The snapshot device is at: /dev/mapper/vgname-md5hash
1222+
String vgNameEscaped = vgName.replace("-", "--");
1223+
String snapshotDevice = vgNameEscaped + "-" + md5Hash;
1224+
1225+
Script dmRemoveCmd = new Script("/usr/sbin/dmsetup", 30000, logger);
1226+
dmRemoveCmd.add("remove");
1227+
dmRemoveCmd.add(snapshotDevice);
1228+
String dmResult = dmRemoveCmd.execute();
1229+
if (dmResult != null) {
1230+
logger.debug("dmsetup remove returned: {} (may already be removed)", dmResult);
1231+
}
1232+
1233+
// Remove the COW (copy-on-write) volume: /dev/vgname/md5hash-cow
1234+
String cowLvPath = "/dev/" + vgName + "/" + md5Hash + "-cow";
1235+
Script removeCowCmd = new Script("/usr/sbin/lvremove", 30000, logger);
1236+
removeCowCmd.add("-f");
1237+
removeCowCmd.add(cowLvPath);
1238+
1239+
String cowResult = removeCowCmd.execute();
1240+
if (cowResult != null) {
1241+
logger.warn("Failed to remove CLVM COW volume {} : {}",cowLvPath, cowResult);
1242+
} else {
1243+
logger.debug("Successfully deleted CLVM snapshot COW volume: {}", cowLvPath);
1244+
}
1245+
1246+
} catch (Exception ex) {
1247+
logger.error("Exception while deleting CLVM snapshot {}", snapshotPath, ex);
1248+
}
1249+
}
1250+
1251+
/**
1252+
* Compute MD5 hash of a string, matching what managesnapshot.sh does:
1253+
* echo "${snapshot}" | md5sum -t | awk '{ print $1 }'
1254+
*/
1255+
private String computeMd5Hash(String input) {
1256+
try {
1257+
java.security.MessageDigest md = java.security.MessageDigest.getInstance("MD5");
1258+
byte[] array = md.digest((input + "\n").getBytes("UTF-8"));
1259+
StringBuilder sb = new StringBuilder();
1260+
for (byte b : array) {
1261+
sb.append(String.format("%02x", b));
1262+
}
1263+
return sb.toString();
1264+
} catch (Exception e) {
1265+
logger.error("Failed to compute MD5 hash for: {}", input, e);
1266+
return input;
1267+
}
1268+
}
1269+
11841270
protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params, DataStoreTO store) throws
11851271
LibvirtException, InternalErrorException {
11861272
DiskDef iso = new DiskDef();
@@ -1842,8 +1928,14 @@ public Answer createSnapshot(final CreateObjectCommand cmd) {
18421928
}
18431929
}
18441930

1845-
if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && volume.requiresEncryption()) {
1846-
throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported");
1931+
if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state)) {
1932+
if (volume.requiresEncryption()) {
1933+
throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported");
1934+
}
1935+
1936+
if (StoragePoolType.CLVM.name().equals(primaryStore.getType())) {
1937+
throw new CloudRuntimeException("VM is running, live snapshots aren't supported with CLVM primary storage");
1938+
}
18471939
}
18481940

18491941
KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java‎

Lines changed: 165 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import com.cloud.agent.properties.AgentProperties;
3636
import com.cloud.agent.properties.AgentPropertiesFileHandler;
37+
import com.cloud.utils.script.OutputInterpreter;
3738
import org.apache.cloudstack.api.ApiConstants;
3839
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
3940
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
@@ -254,9 +255,12 @@ public StorageVol getVolume(StoragePool pool, String volName) {
254255

255256
try {
256257
vol = pool.storageVolLookupByName(volName);
257-
logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool");
258+
if (vol != null) {
259+
logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool");
260+
}
258261
} catch (LibvirtException e) {
259-
throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage());
262+
logger.debug("Volume " + volName + " still not found after pool refresh: " + e.getMessage());
263+
return null;
260264
}
261265
}
262266

@@ -663,6 +667,17 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
663667

664668
try {
665669
StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid);
670+
671+
// Check if volume was found - if null, treat as not found and trigger fallback for CLVM
672+
if (vol == null) {
673+
logger.debug("Volume " + volumeUuid + " not found in libvirt, will check for CLVM fallback");
674+
if (pool.getType() == StoragePoolType.CLVM) {
675+
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
676+
}
677+
678+
throw new CloudRuntimeException("Volume " + volumeUuid + " not found in libvirt pool");
679+
}
680+
666681
KVMPhysicalDisk disk;
667682
LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol);
668683
disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool);
@@ -693,11 +708,153 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
693708
}
694709
return disk;
695710
} catch (LibvirtException e) {
696-
logger.debug("Failed to get physical disk:", e);
711+
logger.debug("Failed to get volume from libvirt: " + e.getMessage());
712+
// For CLVM, try direct block device access as fallback
713+
if (pool.getType() == StoragePoolType.CLVM) {
714+
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
715+
}
716+
697717
throw new CloudRuntimeException(e.toString());
698718
}
699719
}
700720

721+
private KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool, LibvirtStoragePool libvirtPool) {
722+
logger.info("CLVM volume not visible to libvirt, attempting direct block device access for volume: {}", volumeUuid);
723+
724+
try {
725+
logger.debug("Refreshing libvirt storage pool: {}", pool.getUuid());
726+
libvirtPool.getPool().refresh(0);
727+
728+
StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid);
729+
if (vol != null) {
730+
logger.info("Volume found after pool refresh: {}", volumeUuid);
731+
KVMPhysicalDisk disk;
732+
LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol);
733+
disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool);
734+
disk.setSize(vol.getInfo().allocation);
735+
disk.setVirtualSize(vol.getInfo().capacity);
736+
disk.setFormat(voldef.getFormat() == LibvirtStorageVolumeDef.VolumeFormat.QCOW2 ?
737+
PhysicalDiskFormat.QCOW2 : PhysicalDiskFormat.RAW);
738+
return disk;
739+
}
740+
} catch (LibvirtException refreshEx) {
741+
logger.debug("Pool refresh failed or volume still not found: {}", refreshEx.getMessage());
742+
}
743+
744+
// Still not found after refresh, try direct block device access
745+
return getPhysicalDiskViaDirectBlockDevice(volumeUuid, pool);
746+
}
747+
748+
/**
749+
* For CLVM volumes that exist in LVM but are not visible to libvirt,
750+
* access them directly via block device path.
751+
*/
752+
private KVMPhysicalDisk getPhysicalDiskViaDirectBlockDevice(String volumeUuid, KVMStoragePool pool) {
753+
try {
754+
// For CLVM, pool sourceDir contains the VG path (e.g., "/dev/acsvg")
755+
// Extract the VG name
756+
String sourceDir = pool.getLocalPath();
757+
if (sourceDir == null || sourceDir.isEmpty()) {
758+
throw new CloudRuntimeException("CLVM pool sourceDir is not set, cannot determine VG name");
759+
}
760+
761+
String vgName = sourceDir;
762+
if (vgName.startsWith("/")) {
763+
String[] parts = vgName.split("/");
764+
List<String> tokens = Arrays.stream(parts)
765+
.filter(s -> !s.isEmpty()).collect(Collectors.toList());
766+
767+
vgName = tokens.size() > 1 ? tokens.get(1)
768+
: tokens.size() == 1 ? tokens.get(0)
769+
: "";
770+
}
771+
772+
logger.debug("Using VG name: {} (from sourceDir: {}) ", vgName, sourceDir);
773+
774+
// Check if the LV exists in LVM using lvs command
775+
logger.debug("Checking if volume {} exsits in VG {}", volumeUuid, vgName);
776+
Script checkLvCmd = new Script("/usr/sbin/lvs", 5000, logger);
777+
checkLvCmd.add("--noheadings");
778+
checkLvCmd.add("--unbuffered");
779+
checkLvCmd.add(vgName + "/" + volumeUuid);
780+
781+
String checkResult = checkLvCmd.execute();
782+
if (checkResult != null) {
783+
logger.debug("Volume {} does not exist in VG {}: {}", volumeUuid, vgName, checkResult);
784+
throw new CloudRuntimeException(String.format("Storage volume not found: no storage vol with matching name '%s'", volumeUuid));
785+
}
786+
787+
logger.info("Volume {} exists in LVM but not visible to libvirt, accessing directly", volumeUuid);
788+
789+
// Try standard device path first
790+
String lvPath = "/dev/" + vgName + "/" + volumeUuid;
791+
File lvDevice = new File(lvPath);
792+
793+
if (!lvDevice.exists()) {
794+
// Try device-mapper path with escaped hyphens
795+
String vgNameEscaped = vgName.replace("-", "--");
796+
String volumeUuidEscaped = volumeUuid.replace("-", "--");
797+
lvPath = "/dev/mapper/" + vgNameEscaped + "-" + volumeUuidEscaped;
798+
lvDevice = new File(lvPath);
799+
800+
if (!lvDevice.exists()) {
801+
logger.warn("Volume exists in LVM but device node not found: {}", volumeUuid);
802+
throw new CloudRuntimeException(String.format("Could not find volume %s " +
803+
"in VG %s - volume exists in LVM but device node not accessible", volumeUuid, vgName));
804+
}
805+
}
806+
807+
long size = 0;
808+
try {
809+
Script lvsCmd = new Script("/usr/sbin/lvs", 5000, logger);
810+
lvsCmd.add("--noheadings");
811+
lvsCmd.add("--units");
812+
lvsCmd.add("b");
813+
lvsCmd.add("-o");
814+
lvsCmd.add("lv_size");
815+
lvsCmd.add(lvPath);
816+
817+
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
818+
String result = lvsCmd.execute(parser);
819+
820+
String output = null;
821+
if (result == null) {
822+
output = parser.getLines();
823+
} else {
824+
output = result;
825+
}
826+
827+
if (output != null && !output.isEmpty()) {
828+
String sizeStr = output.trim().replaceAll("[^0-9]", "");
829+
if (!sizeStr.isEmpty()) {
830+
size = Long.parseLong(sizeStr);
831+
}
832+
}
833+
} catch (Exception sizeEx) {
834+
logger.warn("Failed to get size for CLVM volume via lvs: {}", sizeEx.getMessage());
835+
if (lvDevice.isFile()) {
836+
size = lvDevice.length();
837+
}
838+
}
839+
840+
KVMPhysicalDisk disk = new KVMPhysicalDisk(lvPath, volumeUuid, pool);
841+
disk.setFormat(PhysicalDiskFormat.RAW);
842+
disk.setSize(size);
843+
disk.setVirtualSize(size);
844+
845+
logger.info("Successfully accessed CLVM volume via direct block device: {} " +
846+
"with size: {} bytes",lvPath, size);
847+
848+
return disk;
849+
850+
} catch (CloudRuntimeException ex) {
851+
throw ex;
852+
} catch (Exception ex) {
853+
logger.error("Failed to access CLVM volume via direct block device: {}",volumeUuid, ex);
854+
throw new CloudRuntimeException(String.format("Could not find volume %s: %s ",volumeUuid, ex.getMessage()));
855+
}
856+
}
857+
701858
/**
702859
* adjust refcount
703860
*/
@@ -1227,7 +1384,11 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag
12271384
LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool;
12281385
try {
12291386
StorageVol vol = getVolume(libvirtPool.getPool(), uuid);
1230-
logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid());
1387+
if (vol == null) {
1388+
logger.warn("Volume %s not found in libvirt pool %s, it may have been already deleted", uuid, pool.getUuid());
1389+
return true;
1390+
}
1391+
logger.debug("Instructing libvirt to remove volume %s from pool %s", uuid, pool.getUuid());
12311392
if(Storage.ImageFormat.DIR.equals(format)){
12321393
deleteDirVol(libvirtPool, vol);
12331394
} else {

‎scripts/storage/qcow2/managesnapshot.sh‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -212,12 +212,12 @@ backup_snapshot() {
212212
return 1
213213
fi
214214

215-
qemuimg_ret=$($qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}")
215+
qemuimg_ret=$($qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" 2>&1)
216216
ret_code=$?
217-
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]
217+
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]])
218218
then
219219
forceShareFlag=""
220-
$qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}"
220+
$qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}"
221221
ret_code=$?
222222
fi
223223
if [ $ret_code -gt 0 ]
@@ -240,9 +240,9 @@ backup_snapshot() {
240240
# Backup VM snapshot
241241
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1)
242242
ret_code=$?
243-
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]; then
243+
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then
244244
forceShareFlag=""
245-
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk)
245+
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1)
246246
ret_code=$?
247247
fi
248248

@@ -251,11 +251,11 @@ backup_snapshot() {
251251
return 1
252252
fi
253253

254-
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
254+
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1)
255255
ret_code=$?
256-
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"convert: invalid option -- 'U'"* ]]; then
256+
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then
257257
forceShareFlag=""
258-
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
258+
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1)
259259
ret_code=$?
260260
fi
261261

0 commit comments

Comments
 (0)