Skip to content

Commit ceb4dee

Browse files
burmanmburmanm
authored
Add hot reloading of SslContext (#376)
* When TLS is enabled, create a Filewatcher to detect any changes in the filesystem related to these TLS files (key, CA, cert). If something happens on the disk to these files, simply reload the SslContext. * Add smallish test for the reload * Change the test to use spy instance * Add logging to indicate that the TLS/SSL certificates are being reloaded
1 parent c86da02 commit ceb4dee

File tree

4 files changed

+151
-14
lines changed

4 files changed

+151
-14
lines changed

‎management-api-server/src/main/java/com/datastax/mgmtapi/Cli.java‎

Lines changed: 94 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,21 @@
4646
import java.io.IOException;
4747
import java.net.SocketAddress;
4848
import java.net.URI;
49+
import java.nio.file.FileSystems;
4950
import java.nio.file.Files;
5051
import java.nio.file.InvalidPathException;
5152
import java.nio.file.Paths;
53+
import java.nio.file.StandardWatchEventKinds;
54+
import java.nio.file.WatchEvent;
55+
import java.nio.file.WatchKey;
56+
import java.nio.file.WatchService;
5257
import java.util.ArrayList;
5358
import java.util.Collection;
5459
import java.util.Collections;
5560
import java.util.List;
5661
import java.util.Optional;
5762
import java.util.concurrent.CountDownLatch;
63+
import java.util.concurrent.ExecutorService;
5864
import java.util.concurrent.Executors;
5965
import java.util.concurrent.ScheduledExecutorService;
6066
import java.util.concurrent.ScheduledFuture;
@@ -74,7 +80,6 @@
7480
name = "cassandra-management-api",
7581
description = "REST service for managing an Apache Cassandra or DSE node")
7682
public class Cli implements Runnable {
77-
public static final String PROTOCOL_TLS_V1_2 = "TLSv1.2";
7883

7984
static {
8085
InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory());
@@ -136,20 +141,26 @@ public class Cli implements Runnable {
136141
description = "Path to trust certs signed only by this CA")
137142
private String tls_ca_cert_file;
138143

144+
private File tlsCaCert;
145+
139146
@Path(writable = false)
140147
@Option(
141148
name = {"--tlscert"},
142149
arity = 1,
143150
description = "Path to TLS certificate file")
144151
private String tls_cert_file;
145152

153+
private File tlsCert;
154+
146155
@Path(writable = false)
147156
@Option(
148157
name = {"--tlskey"},
149158
arity = 1,
150159
description = "Path to TLS key file")
151160
private String tls_key_file;
152161

162+
private File tlsKey;
163+
153164
private boolean useTls = false;
154165
private File dbUnixSocketFile = null;
155166
private File dbHomeDir = null;
@@ -159,6 +170,8 @@ public class Cli implements Runnable {
159170
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
160171
private List<NettyJaxrsServer> servers = new ArrayList<>();
161172

173+
private SslContext sslContext;
174+
162175
public Cli() {}
163176

164177
@VisibleForTesting
@@ -387,6 +400,7 @@ void checkTLSDeps() {
387400
logger.error("Specified CA Cert file does not exist: {}", tls_ca_cert_file);
388401
System.exit(10);
389402
}
403+
tlsCaCert = new File(tls_ca_cert_file);
390404
}
391405

392406
// CERT File Checks
@@ -406,6 +420,7 @@ void checkTLSDeps() {
406420
logger.error("Specified Cert file does not exist: {}", tls_cert_file);
407421
System.exit(13);
408422
}
423+
tlsCert = new File(tls_cert_file);
409424
}
410425

411426
// KEY File Checks
@@ -424,6 +439,7 @@ void checkTLSDeps() {
424439
logger.error("Specified Key file does not exist: {}", tls_key_file);
425440
System.exit(16);
426441
}
442+
tlsKey = new File(tls_key_file);
427443
}
428444

429445
useTls = hasAny;
@@ -436,18 +452,86 @@ void preflightChecks() {
436452
checkUnixSocket();
437453
}
438454

439-
private NettyJaxrsServer startHTTPService(String hostname, int port) throws SSLException {
455+
@VisibleForTesting
456+
void createSSLContext() throws SSLException {
457+
this.sslContext =
458+
SslContextBuilder.forServer(tlsCert, tlsKey)
459+
.trustManager(tlsCaCert)
460+
.clientAuth(ClientAuth.REQUIRE)
461+
.ciphers(null, IdentityCipherSuiteFilter.INSTANCE)
462+
.build();
463+
}
464+
465+
@VisibleForTesting
466+
void createSSLWatcher() throws IOException {
467+
// Watch for tls_cert_file, tls_key_file and tls_ca_cert_file, add all their directories to
468+
// Filesystem Watcher
469+
WatchService watchService = FileSystems.getDefault().newWatchService();
470+
java.nio.file.Path tlsCertParent = tlsCert.toPath().getParent();
471+
java.nio.file.Path tlsKeyParent = tlsKey.toPath().getParent();
472+
java.nio.file.Path tlsCaCertParent = tlsCaCert.toPath().getParent();
473+
474+
tlsCertParent.register(
475+
watchService,
476+
StandardWatchEventKinds.ENTRY_CREATE,
477+
StandardWatchEventKinds.ENTRY_DELETE,
478+
StandardWatchEventKinds.ENTRY_MODIFY);
479+
tlsKeyParent.register(
480+
watchService,
481+
StandardWatchEventKinds.ENTRY_CREATE,
482+
StandardWatchEventKinds.ENTRY_DELETE,
483+
StandardWatchEventKinds.ENTRY_MODIFY);
484+
tlsCaCertParent.register(
485+
watchService,
486+
StandardWatchEventKinds.ENTRY_CREATE,
487+
StandardWatchEventKinds.ENTRY_DELETE,
488+
StandardWatchEventKinds.ENTRY_MODIFY);
489+
490+
ExecutorService executorService = Executors.newSingleThreadExecutor();
491+
executorService.execute(
492+
() -> {
493+
while (true) {
494+
try {
495+
WatchKey key = watchService.take();
496+
List<WatchEvent<?>> events = key.pollEvents();
497+
boolean reloadNeeded = false;
498+
for (WatchEvent<?> event : events) {
499+
WatchEvent.Kind<?> kind = event.kind();
500+
501+
WatchEvent<java.nio.file.Path> ev = (WatchEvent<java.nio.file.Path>) event;
502+
java.nio.file.Path eventFilename = ev.context();
503+
504+
if (tlsCertParent.resolve(eventFilename).equals(tlsCert.toPath())
505+
|| tlsKeyParent.resolve(eventFilename).equals(tlsKey.toPath())
506+
|| tlsCaCertParent.resolve(eventFilename).equals(tlsCaCert)) {
507+
// Something in the TLS has been modified.. recreate SslContext
508+
reloadNeeded = true;
509+
}
510+
}
511+
if (!key.reset()) {
512+
// The watched directories have disappeared..
513+
break;
514+
}
515+
if (reloadNeeded) {
516+
logger.info("Detected change in the SSL/TLS certificates, reloading.");
517+
createSSLContext();
518+
}
519+
} catch (InterruptedException e) {
520+
// Do something.. just log?
521+
logger.error("Filesystem watcher received InterruptedException", e);
522+
} catch (IOException e) {
523+
logger.error("Filesystem watcher received IOException", e);
524+
}
525+
}
526+
});
527+
}
528+
529+
private NettyJaxrsServer startHTTPService(String hostname, int port) throws IOException {
440530
NettyJaxrsServer server;
441531

442532
if (useTls) {
443-
SslContext sslContext =
444-
SslContextBuilder.forServer(new File(tls_cert_file), new File(tls_key_file))
445-
.trustManager(new File(tls_ca_cert_file))
446-
.clientAuth(ClientAuth.REQUIRE)
447-
.protocols(PROTOCOL_TLS_V1_2)
448-
.ciphers(null, IdentityCipherSuiteFilter.INSTANCE)
449-
.build();
450-
533+
createSSLContext();
534+
createSSLWatcher();
451535
server = new NettyJaxrsTLSServer(sslContext);
452536
} else {
453537
server = new NettyJaxrsServer();

‎management-api-server/src/test/java/com/datastax/mgmtapi/NettyTlsClientAuthTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import static org.junit.Assert.assertFalse;
1010
import static org.junit.Assert.assertTrue;
1111
import static org.junit.Assume.assumeTrue;
12+
import static org.mockito.Mockito.times;
13+
import static org.mockito.Mockito.verify;
1214

1315
import com.datastax.mgmtapi.helpers.IntegrationTestUtils;
1416
import com.datastax.mgmtapi.helpers.NettyHttpClient;
@@ -41,6 +43,8 @@
4143
import java.io.IOException;
4244
import java.net.URI;
4345
import java.net.URL;
46+
import java.nio.file.Files;
47+
import java.nio.file.Path;
4448
import java.util.List;
4549
import java.util.concurrent.CountDownLatch;
4650
import java.util.concurrent.TimeUnit;
@@ -56,6 +60,7 @@
5660
import org.junit.ClassRule;
5761
import org.junit.Test;
5862
import org.junit.rules.TemporaryFolder;
63+
import org.mockito.Mockito;
5964

6065
public class NettyTlsClientAuthTest {
6166
static String BASE_URI = generateURL("");
@@ -391,4 +396,56 @@ public void testManagementAPIWithTLS() throws IOException {
391396
FileUtils.deleteQuietly(new File(mgmtSock));
392397
}
393398
}
399+
400+
@Test
401+
public void testHotReload() throws Exception {
402+
assumeTrue(IntegrationTestUtils.shouldRun());
403+
404+
String mgmtSock = SocketUtils.makeValidUnixSocketFile(null, "management-netty-tls-mgmt");
405+
new File(mgmtSock).deleteOnExit();
406+
String cassSock = SocketUtils.makeValidUnixSocketFile(null, "management-netty-tls-cass");
407+
new File(cassSock).deleteOnExit();
408+
409+
Path tempDirectory = Files.createTempDirectory("reload-test");
410+
411+
List<String> extraArgs =
412+
IntegrationTestUtils.getExtraArgs(
413+
NettyTlsClientAuthTest.class, "", temporaryFolder.getRoot());
414+
415+
File trustCertFile =
416+
IntegrationTestUtils.getFile(getClass(), "mutual_auth_client_cert_chain.pem");
417+
File serverKeyFile = IntegrationTestUtils.getFile(getClass(), "mutual_auth_server.key");
418+
File serverCrtFile = IntegrationTestUtils.getFile(getClass(), "mutual_auth_server.crt");
419+
420+
// Copy TLS files to temp directory
421+
Path trustCertCopy =
422+
Files.copy(trustCertFile.toPath(), tempDirectory.resolve(trustCertFile.toPath()));
423+
Path keyCopy =
424+
Files.copy(serverKeyFile.toPath(), tempDirectory.resolve(serverKeyFile.toPath()));
425+
Path crtCopy =
426+
Files.copy(serverCrtFile.toPath(), tempDirectory.resolve(serverCrtFile.toPath()));
427+
428+
Cli cli =
429+
new Cli(
430+
Lists.newArrayList("file://" + mgmtSock, BASE_URI),
431+
IntegrationTestUtils.getCassandraHome(),
432+
cassSock,
433+
false,
434+
extraArgs,
435+
trustCertCopy.toFile().getAbsolutePath(),
436+
crtCopy.toFile().getAbsolutePath(),
437+
keyCopy.toFile().getAbsolutePath());
438+
cli.preflightChecks();
439+
Cli spy = Mockito.spy(cli);
440+
spy.createSSLContext();
441+
spy.createSSLWatcher();
442+
443+
verify(spy, times(1)).createSSLContext();
444+
verify(spy, times(1)).createSSLWatcher();
445+
446+
// Modify files..
447+
Files.copy(trustCertFile.toPath(), tempDirectory.resolve(trustCertFile.toPath()));
448+
449+
verify(spy, Mockito.timeout(1000)).createSSLContext();
450+
}
394451
}

‎management-api-server/src/test/java/com/datastax/mgmtapi/helpers/NettyHttpClient.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import static org.junit.Assert.fail;
99

10-
import com.datastax.mgmtapi.Cli;
1110
import com.google.common.util.concurrent.Uninterruptibles;
1211
import io.netty.bootstrap.Bootstrap;
1312
import io.netty.channel.Channel;
@@ -59,7 +58,6 @@ public NettyHttpClient(URL endpoint, File clientCaFile, File clientCertFile, Fil
5958
.trustManager(clientCaFile)
6059
.keyManager(clientCertFile, clientKeyFile, null)
6160
.ciphers(null, IdentityCipherSuiteFilter.INSTANCE)
62-
.protocols(Cli.PROTOCOL_TLS_V1_2)
6361
.sessionCacheSize(0)
6462
.sessionTimeout(0)
6563
.build();

‎management-api-server/src/test/java/com/datastax/mgmtapi/helpers/NettyHttpIPCClient.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import static org.junit.Assert.fail;
99

10-
import com.datastax.mgmtapi.Cli;
1110
import com.datastax.mgmtapi.ipc.IPCController;
1211
import com.google.common.util.concurrent.Uninterruptibles;
1312
import io.netty.channel.Channel;
@@ -63,7 +62,6 @@ public NettyHttpIPCClient(
6362
SslContextBuilder.forClient()
6463
.trustManager(clientCaFile)
6564
.keyManager(clientCertFile, clientKeyFile, null)
66-
.protocols(Cli.PROTOCOL_TLS_V1_2)
6765
.ciphers(null, IdentityCipherSuiteFilter.INSTANCE)
6866
.sessionCacheSize(0)
6967
.sessionTimeout(0)

0 commit comments

Comments
 (0)