Skip to content

Commit 48ffa5d

Browse files
weizhouapacheweizhouapache
authored
Support multiple ceph monitors (#6792)
1 parent 17fe984 commit 48ffa5d

File tree

14 files changed

+539
-45
lines changed

14 files changed

+539
-45
lines changed

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public boolean parseDomainXML(String domXML) {
8383
String protocol = getAttrValue("source", "protocol", disk);
8484
String authUserName = getAttrValue("auth", "username", disk);
8585
String poolUuid = getAttrValue("secret", "uuid", disk);
86-
String host = getAttrValue("host", "name", disk);
86+
String host = LibvirtStoragePoolXMLParser.getStorageHosts(disk);
8787
int port = 0;
8888
String xmlPort = getAttrValue("host", "port", disk);
8989
if (StringUtils.isNotBlank(xmlPort)) {

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,10 +147,14 @@ public String toString() {
147147
}
148148
if (_poolType == PoolType.RBD) {
149149
storagePoolBuilder.append("<source>\n");
150-
if (_sourcePort > 0) {
151-
storagePoolBuilder.append("<host name='" + _sourceHost + "' port='" + _sourcePort + "'/>\n");
152-
} else {
153-
storagePoolBuilder.append("<host name='" + _sourceHost + "'/>\n");
150+
for (String sourceHost : _sourceHost.split(",")) {
151+
storagePoolBuilder.append("<host name='");
152+
storagePoolBuilder.append(sourceHost);
153+
if (_sourcePort != 0) {
154+
storagePoolBuilder.append("' port='");
155+
storagePoolBuilder.append(_sourcePort);
156+
}
157+
storagePoolBuilder.append("'/>\n");
154158
}
155159

156160
storagePoolBuilder.append("<name>" + _sourceDir + "</name>\n");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
import java.io.IOException;
2020
import java.io.StringReader;
21+
import java.util.ArrayList;
22+
import java.util.List;
2123

2224
import javax.xml.parsers.DocumentBuilder;
2325
import javax.xml.parsers.ParserConfigurationException;
@@ -52,7 +54,7 @@ public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) {
5254
String poolName = getTagValue("name", rootElement);
5355

5456
Element source = (Element)rootElement.getElementsByTagName("source").item(0);
55-
String host = getAttrValue("host", "name", source);
57+
String host = getStorageHosts(source);
5658
String format = getAttrValue("format", "type", source);
5759

5860
if (type.equalsIgnoreCase("rbd") || type.equalsIgnoreCase("powerflex")) {
@@ -123,4 +125,13 @@ private static String getAttrValue(String tag, String attr, Element eElement) {
123125
Element node = (Element)tagNode.item(0);
124126
return node.getAttribute(attr);
125127
}
128+
129+
protected static String getStorageHosts(Element parentElement) {
130+
List<String> storageHosts = new ArrayList<>();
131+
NodeList hosts = parentElement.getElementsByTagName("host");
132+
for (int j = 0; j < hosts.getLength(); j++) {
133+
storageHosts.add(((Element) hosts.item(j)).getAttribute("name"));
134+
}
135+
return StringUtils.join(storageHosts, ",");
136+
}
126137
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1093,13 +1093,15 @@ public String toString() {
10931093
diskBuilder.append(" protocol='" + _diskProtocol + "'");
10941094
diskBuilder.append(" name='" + _sourcePath + "'");
10951095
diskBuilder.append(">\n");
1096-
diskBuilder.append("<host name='");
1097-
diskBuilder.append(_sourceHost);
1098-
if (_sourcePort != 0) {
1099-
diskBuilder.append("' port='");
1100-
diskBuilder.append(_sourcePort);
1096+
for (String sourceHost : _sourceHost.split(",")) {
1097+
diskBuilder.append("<host name='");
1098+
diskBuilder.append(sourceHost.replace("[", "").replace("]", ""));
1099+
if (_sourcePort != 0) {
1100+
diskBuilder.append("' port='");
1101+
diskBuilder.append(_sourcePort);
1102+
}
1103+
diskBuilder.append("'/>\n");
11011104
}
1102-
diskBuilder.append("'/>\n");
11031105
diskBuilder.append("</source>\n");
11041106
if (_authUserName != null) {
11051107
diskBuilder.append("<auth username='" + _authUserName + "'>\n");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818

1919
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
2020
import org.apache.cloudstack.utils.qemu.QemuObject;
21+
import org.apache.commons.lang3.StringUtils;
22+
23+
import java.util.ArrayList;
24+
import java.util.List;
2125

2226
public class KVMPhysicalDisk {
2327
private String path;
@@ -29,10 +33,7 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
2933
String rbdOpts;
3034

3135
rbdOpts = "rbd:" + image;
32-
rbdOpts += ":mon_host=" + monHost;
33-
if (monPort > 0) {
34-
rbdOpts += "\\:" + monPort;
35-
}
36+
rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort);
3637

3738
if (authUserName == null) {
3839
rbdOpts += ":auth_supported=none";
@@ -48,6 +49,25 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
4849
return rbdOpts;
4950
}
5051

52+
private static String composeOptionForMonHosts(String monHost, int monPort) {
53+
List<String> hosts = new ArrayList<>();
54+
for (String host : monHost.split(",")) {
55+
if (monPort > 0) {
56+
hosts.add(replaceHostAddress(host) + "\\:" + monPort);
57+
} else {
58+
hosts.add(replaceHostAddress(host));
59+
}
60+
}
61+
return StringUtils.join(hosts, "\\;");
62+
}
63+
64+
private static String replaceHostAddress(String hostIp) {
65+
if (hostIp != null && hostIp.startsWith("[") && hostIp.endsWith("]")) {
66+
return hostIp.replaceAll("\\:", "\\\\:");
67+
}
68+
return hostIp;
69+
}
70+
5171
private PhysicalDiskFormat format;
5272
private long size;
5373
private long virtualSize;

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import junit.framework.TestCase;
2323
import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.PoolType;
2424
import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.AuthenticationType;
25+
import org.junit.Test;
2526

2627
public class LibvirtStoragePoolDefTest extends TestCase {
2728

@@ -102,4 +103,35 @@ public void testRbdStoragePoolWithoutPort() {
102103

103104
assertEquals(expectedXml, pool.toString());
104105
}
106+
107+
@Test
108+
public void testRbdStoragePoolWithMultipleHostsIpv6() {
109+
PoolType type = PoolType.RBD;
110+
String name = "myRBDPool";
111+
String uuid = "1583a25a-b192-436c-93e6-0ef60b198a32";
112+
String host = "[fc00:1234::1],[fc00:1234::2],[fc00:1234::3]";
113+
int port = 3300;
114+
String authUsername = "admin";
115+
AuthenticationType auth = AuthenticationType.CEPH;
116+
String dir = "rbd";
117+
String secretUuid = "28909c4f-314e-4db7-a6b3-5eccd9dcf973";
118+
119+
LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, port, dir, authUsername, auth, secretUuid);
120+
121+
String expected = "<pool type='rbd'>\n" +
122+
"<name>myRBDPool</name>\n" +
123+
"<uuid>1583a25a-b192-436c-93e6-0ef60b198a32</uuid>\n" +
124+
"<source>\n" +
125+
"<host name='[fc00:1234::1]' port='3300'/>\n" +
126+
"<host name='[fc00:1234::2]' port='3300'/>\n" +
127+
"<host name='[fc00:1234::3]' port='3300'/>\n" +
128+
"<name>rbd</name>\n" +
129+
"<auth username='admin' type='ceph'>\n" +
130+
"<secret uuid='28909c4f-314e-4db7-a6b3-5eccd9dcf973'/>\n" +
131+
"</auth>\n" +
132+
"</source>\n" +
133+
"</pool>\n";
134+
135+
assertEquals(expected, pool.toString());
136+
}
105137
}

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package com.cloud.hypervisor.kvm.resource;
21+
22+
import junit.framework.TestCase;
23+
import org.junit.Assert;
24+
25+
public class LibvirtStoragePoolXMLParserTest extends TestCase {
26+
27+
public void testParseNfsStoragePoolXML() {
28+
String poolXML = "<pool type='netfs'>\n" +
29+
" <name>feff06b5-84b2-3258-b5f9-1953217295de</name>\n" +
30+
" <uuid>feff06b5-84b2-3258-b5f9-1953217295de</uuid>\n" +
31+
" <capacity unit='bytes'>111111111</capacity>\n" +
32+
" <allocation unit='bytes'>2222222</allocation>\n" +
33+
" <available unit='bytes'>3333333</available>\n" +
34+
" <source>\n" +
35+
" <host name='10.11.12.13'/>\n" +
36+
" <dir path='/mnt/primary1'/>\n" +
37+
" <format type='auto'/>\n" +
38+
" </source>\n" +
39+
" <target>\n" +
40+
" <path>/mnt/feff06b5-84b2-3258-b5f9-1953217295de</path>\n" +
41+
" <permissions>\n" +
42+
" <mode>0755</mode>\n" +
43+
" <owner>0</owner>\n" +
44+
" <group>0</group>\n" +
45+
" </permissions>\n" +
46+
" </target>\n" +
47+
"</pool>";
48+
49+
LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser();
50+
LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML);
51+
52+
Assert.assertEquals("10.11.12.13", pool.getSourceHost());
53+
}
54+
55+
public void testParseRbdStoragePoolXMLWithMultipleHosts() {
56+
String poolXML = "<pool type='rbd'>\n" +
57+
" <name>feff06b5-84b2-3258-b5f9-1953217295de</name>\n" +
58+
" <uuid>feff06b5-84b2-3258-b5f9-1953217295de</uuid>\n" +
59+
" <source>\n" +
60+
" <name>rbdpool</name>\n" +
61+
" <host name='10.11.12.13' port='6789'/>\n" +
62+
" <host name='10.11.12.14' port='6789'/>\n" +
63+
" <host name='10.11.12.15' port='6789'/>\n" +
64+
" <format type='auto'/>\n" +
65+
" <auth username='admin' type='ceph'>\n" +
66+
" <secret uuid='262f743a-3726-11ed-aaee-93e90b39d5c4'/>\n" +
67+
" </auth>\n" +
68+
" </source>\n" +
69+
"</pool>";
70+
71+
LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser();
72+
LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML);
73+
74+
Assert.assertEquals(LibvirtStoragePoolDef.PoolType.RBD, pool.getPoolType());
75+
Assert.assertEquals(LibvirtStoragePoolDef.AuthenticationType.CEPH, pool.getAuthType());
76+
Assert.assertEquals("10.11.12.13,10.11.12.14,10.11.12.15", pool.getSourceHost());
77+
Assert.assertEquals(6789, pool.getSourcePort());
78+
}
79+
80+
public void testParseRbdStoragePoolXMLWithMultipleHostsIpv6() {
81+
String poolXML = "<pool type='rbd'>\n" +
82+
" <name>feff06b5-84b2-3258-b5f9-1953217295de</name>\n" +
83+
" <uuid>feff06b5-84b2-3258-b5f9-1953217295de</uuid>\n" +
84+
" <source>\n" +
85+
" <name>rbdpool</name>\n" +
86+
" <host name='[fc00:aa:bb:cc::1]' port='6789'/>\n" +
87+
" <host name='[fc00:aa:bb:cc::2]' port='6789'/>\n" +
88+
" <host name='[fc00:aa:bb:cc::3]' port='6789'/>\n" +
89+
" <format type='auto'/>\n" +
90+
" <auth username='admin' type='ceph'>\n" +
91+
" <secret uuid='262f743a-3726-11ed-aaee-93e90b39d5c4'/>\n" +
92+
" </auth>\n" +
93+
" </source>\n" +
94+
"</pool>";
95+
96+
LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser();
97+
LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML);
98+
99+
Assert.assertEquals(LibvirtStoragePoolDef.PoolType.RBD, pool.getPoolType());
100+
Assert.assertEquals(LibvirtStoragePoolDef.AuthenticationType.CEPH, pool.getAuthType());
101+
Assert.assertEquals("[fc00:aa:bb:cc::1],[fc00:aa:bb:cc::2],[fc00:aa:bb:cc::3]", pool.getSourceHost());
102+
Assert.assertEquals(6789, pool.getSourcePort());
103+
}
104+
}

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDefTest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,72 @@ public void testDiskDefWithEncryption() {
238238
assertEquals(disk.toString(), expectedXML);
239239
}
240240

241+
@Test
242+
public void testDiskDefWithMultipleHosts() {
243+
String path = "/mnt/primary1";
244+
String host = "10.11.12.13,10.11.12.14,10.11.12.15";
245+
int port = 3300;
246+
String authUsername = "admin";
247+
String uuid = "40b3f216-36b5-11ed-9357-9b4e21b0ed91";
248+
int devId = 2;
249+
250+
DiskDef diskdef = new DiskDef();
251+
diskdef.defNetworkBasedDisk(path, host, port, authUsername,
252+
uuid, devId, DiskDef.DiskBus.VIRTIO, DiskDef.DiskProtocol.RBD, DiskDef.DiskFmtType.RAW);
253+
254+
assertEquals(path, diskdef.getDiskPath());
255+
assertEquals(DiskDef.DiskType.NETWORK, diskdef.getDiskType());
256+
assertEquals(DiskDef.DiskFmtType.RAW, diskdef.getDiskFormatType());
257+
258+
String expected = "<disk device='disk' type='network'>\n" +
259+
"<driver name='qemu' type='raw' cache='none' />\n" +
260+
"<source protocol='rbd' name='/mnt/primary1'>\n" +
261+
"<host name='10.11.12.13' port='3300'/>\n" +
262+
"<host name='10.11.12.14' port='3300'/>\n" +
263+
"<host name='10.11.12.15' port='3300'/>\n" +
264+
"</source>\n" +
265+
"<auth username='admin'>\n" +
266+
"<secret type='ceph' uuid='40b3f216-36b5-11ed-9357-9b4e21b0ed91'/>\n" +
267+
"</auth>\n" +
268+
"<target dev='vdc' bus='virtio'/>\n" +
269+
"</disk>\n";
270+
271+
assertEquals(expected, diskdef.toString());
272+
}
273+
274+
@Test
275+
public void testDiskDefWithMultipleHostsIpv6() {
276+
String path = "/mnt/primary1";
277+
String host = "[fc00:1234::1],[fc00:1234::2],[fc00:1234::3]";
278+
int port = 3300;
279+
String authUsername = "admin";
280+
String uuid = "40b3f216-36b5-11ed-9357-9b4e21b0ed91";
281+
int devId = 2;
282+
283+
DiskDef diskdef = new DiskDef();
284+
diskdef.defNetworkBasedDisk(path, host, port, authUsername,
285+
uuid, devId, DiskDef.DiskBus.VIRTIO, DiskDef.DiskProtocol.RBD, DiskDef.DiskFmtType.RAW);
286+
287+
assertEquals(path, diskdef.getDiskPath());
288+
assertEquals(DiskDef.DiskType.NETWORK, diskdef.getDiskType());
289+
assertEquals(DiskDef.DiskFmtType.RAW, diskdef.getDiskFormatType());
290+
291+
String expected = "<disk device='disk' type='network'>\n" +
292+
"<driver name='qemu' type='raw' cache='none' />\n" +
293+
"<source protocol='rbd' name='/mnt/primary1'>\n" +
294+
"<host name='fc00:1234::1' port='3300'/>\n" +
295+
"<host name='fc00:1234::2' port='3300'/>\n" +
296+
"<host name='fc00:1234::3' port='3300'/>\n" +
297+
"</source>\n" +
298+
"<auth username='admin'>\n" +
299+
"<secret type='ceph' uuid='40b3f216-36b5-11ed-9357-9b4e21b0ed91'/>\n" +
300+
"</auth>\n" +
301+
"<target dev='vdc' bus='virtio'/>\n" +
302+
"</disk>\n";
303+
304+
assertEquals(expected, diskdef.toString());
305+
}
306+
241307
@Test
242308
public void testDiskDefWithBurst() {
243309
String filePath = "/var/lib/libvirt/images/disk.qcow2";

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDiskTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,26 @@ public void testRBDStringBuilder() {
2828
"rbd:volume1:mon_host=ceph-monitor\\:8000:auth_supported=cephx:id=admin:key=supersecret:rbd_default_format=2:client_mount_timeout=30");
2929
}
3030

31+
public void testRBDStringBuilder2() {
32+
String monHosts = "ceph-monitor1,ceph-monitor2,ceph-monitor3";
33+
int monPort = 3300;
34+
String expected = "rbd:volume1:" +
35+
"mon_host=ceph-monitor1\\:3300\\;ceph-monitor2\\:3300\\;ceph-monitor3\\:3300:" +
36+
"auth_supported=cephx:id=admin:key=supersecret:rbd_default_format=2:client_mount_timeout=30";
37+
String actualResult = KVMPhysicalDisk.RBDStringBuilder(monHosts, monPort, "admin", "supersecret", "volume1");
38+
assertEquals(expected, actualResult);
39+
}
40+
41+
public void testRBDStringBuilder3() {
42+
String monHosts = "[fc00:1234::1],[fc00:1234::2],[fc00:1234::3]";
43+
int monPort = 3300;
44+
String expected = "rbd:volume1:" +
45+
"mon_host=[fc00\\:1234\\:\\:1]\\:3300\\;[fc00\\:1234\\:\\:2]\\:3300\\;[fc00\\:1234\\:\\:3]\\:3300:" +
46+
"auth_supported=cephx:id=admin:key=supersecret:rbd_default_format=2:client_mount_timeout=30";
47+
String actualResult = KVMPhysicalDisk.RBDStringBuilder(monHosts, monPort, "admin", "supersecret", "volume1");
48+
assertEquals(expected, actualResult);
49+
}
50+
3151
public void testAttributes() {
3252
String name = "3bc186e0-6c29-45bf-b2b0-ddef6f91f5ef";
3353
String path = "/" + name;

0 commit comments

Comments
 (0)