diff --git a/src/p11_load.c b/src/p11_load.c
index d3888edb..55ddcd9e 100644
--- a/src/p11_load.c
+++ b/src/p11_load.c
@@ -144,7 +144,7 @@ void pkcs11_CTX_unload(PKCS11_CTX *ctx)
 {
 	PKCS11_CTX_private *cpriv = PRIVCTX(ctx);
 
-	if (!cpriv->method) /* Module not loaded */
+	if (!cpriv->method || !cpriv->handle) /* Module not loaded */
 		return;
 
 	/* Tell the PKCS11 library to shut down */