feat(NODE-3633): add Socks5 support (#3041)

Author: addaleax

.evergreen/config.yml

@@ -293,6 +293,54 @@ functions:
           export NODE_LTS_NAME='${NODE_LTS_NAME}'
 
           bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
+  run socks5 tests:
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: src
+        script: |
+          ${PREPARE_SHELL}
+          cat <<EOT > prepare_client_encryption.sh
+          export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
+          export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
+          export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+          export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
+          export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
+          EOT
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        script: >
+          ${PREPARE_SHELL}
+
+
+          export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo
+          "/opt/mongodbtoolchain/v3/bin/python3")
+
+          export PROJECT_DIRECTORY="$(pwd)"
+
+          export DRIVERS_TOOLS="${DRIVERS_TOOLS}"
+
+          export NODE_LTS_NAME='${NODE_LTS_NAME}'
+
+          export MONGODB_URI="${MONGODB_URI}"
+
+          export SSL="${SSL}"
+
+
+          # Disable xtrace (just in case it was accidentally set).
+
+          set +x
+
+          . ./prepare_client_encryption.sh
+
+          rm -f ./prepare_client_encryption.sh
+
+
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh
   run kerberos tests:
     - command: shell.exec
       type: test
@@ -907,6 +955,27 @@ tasks:
     commands:
       - func: install dependencies
       - func: run ldap tests
+  - name: test-socks5
+    tags: []
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: replica_set
+      - func: run socks5 tests
+  - name: test-socks5-tls
+    tags: []
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          SSL: ssl
+          VERSION: latest
+          TOPOLOGY: replica_set
+      - func: run socks5 tests
+        vars:
+          SSL: ssl
   - name: test-ocsp-valid-cert-server-staples
     tags:
       - ocsp
@@ -1683,6 +1752,8 @@ buildvariants:
       - test-atlas-data-lake
       - test-auth-kerberos
       - test-auth-ldap
+      - test-socks5
+      - test-socks5-tls
       - test-ocsp-valid-cert-server-staples
       - test-ocsp-invalid-cert-server-staples
       - test-ocsp-valid-cert-server-does-not-staple
@@ -1753,6 +1824,8 @@ buildvariants:
       - test-load-balancer
       - test-auth-kerberos
       - test-auth-ldap
+      - test-socks5
+      - test-socks5-tls
       - test-ocsp-valid-cert-server-staples
       - test-ocsp-invalid-cert-server-staples
       - test-ocsp-valid-cert-server-does-not-staple
@@ -1819,6 +1892,8 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-data-lake
+      - test-socks5
+      - test-socks5-tls
       - test-ocsp-valid-cert-server-staples
       - test-ocsp-invalid-cert-server-staples
       - test-ocsp-valid-cert-server-does-not-staple

.evergreen/config.yml.in

@@ -326,6 +326,43 @@ functions:
 
           bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh
 
+  "run socks5 tests":
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          cat <<EOT > prepare_client_encryption.sh
+          export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION}
+          export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
+          export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
+          export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+          export CSFLE_GIT_REF="${CSFLE_GIT_REF}"
+          export CDRIVER_GIT_REF="${CDRIVER_GIT_REF}"
+          EOT
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+
+          export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo "/opt/mongodbtoolchain/v3/bin/python3")
+          export PROJECT_DIRECTORY="$(pwd)"
+          export DRIVERS_TOOLS="${DRIVERS_TOOLS}"
+          export NODE_LTS_NAME='${NODE_LTS_NAME}'
+          export MONGODB_URI="${MONGODB_URI}"
+          export SSL="${SSL}"
+
+          # Disable xtrace (just in case it was accidentally set).
+          set +x
+          . ./prepare_client_encryption.sh
+          rm -f ./prepare_client_encryption.sh
+
+          bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh
+
   "run kerberos tests":
     - command: shell.exec
       type: test

.evergreen/generate_evergreen_tasks.js

@@ -136,6 +136,37 @@ TASKS.push(
       tags: ['auth', 'ldap'],
       commands: [{ func: 'install dependencies' }, { func: 'run ldap tests' }]
     },
+    {
+      name: 'test-socks5',
+      tags: [],
+      commands: [
+        { func: 'install dependencies' },
+        {
+          func: 'bootstrap mongo-orchestration',
+          vars: {
+            VERSION: 'latest',
+            TOPOLOGY: 'replica_set'
+          }
+        },
+        { func: 'run socks5 tests' }
+      ]
+    },
+    {
+      name: 'test-socks5-tls',
+      tags: [],
+      commands: [
+        { func: 'install dependencies' },
+        {
+          func: 'bootstrap mongo-orchestration',
+          vars: {
+            SSL: 'ssl',
+            VERSION: 'latest',
+            TOPOLOGY: 'replica_set'
+          }
+        },
+        { func: 'run socks5 tests', vars: { SSL: 'ssl' } }
+      ]
+    },
     {
       name: 'test-ocsp-valid-cert-server-staples',
       tags: ['ocsp'],

.evergreen/run-custom-csfle-tests.sh

@@ -1,5 +1,7 @@
 #! /usr/bin/env bash
 
+set +o xtrace # Do not write AWS credentials to stderr
+
 # Initiail checks for running these tests
 if [ -z ${AWS_ACCESS_KEY_ID+omitted} ]; then echo "AWS_ACCESS_KEY_ID is unset" && exit 1; fi
 if [ -z ${AWS_SECRET_ACCESS_KEY+omitted} ]; then echo "AWS_SECRET_ACCESS_KEY is unset" && exit 1; fi
@@ -38,12 +40,14 @@ git clone https://github.com/mongodb/libmongocrypt.git
 pushd libmongocrypt
 git fetch --tags
 git checkout "$CSFLE_GIT_REF" -b csfle-custom
+echo "checked out libmongocrypt at $(git rev-parse HEAD)"
 popd # libmongocrypt
 
 git clone https://github.com/mongodb/mongo-c-driver.git
 pushd mongo-c-driver
 git fetch --tags
 git checkout "$CDRIVER_GIT_REF" -b cdriver-custom
+echo "checked out C driver at $(git rev-parse HEAD)"
 popd # mongo-c-driver
 
 pushd libmongocrypt/bindings/node

.evergreen/run-socks5-tests.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
+
+set -o errexit  # Exit the script with error if any of the commands fail
+set -o xtrace  # For debuggability, no external credentials are used here
+
+PYTHON_BINARY=${PYTHON_BINARY:-python3}
+
+# ssl setup
+SSL=${SSL:-nossl}
+if [ "$SSL" != "nossl" ]; then
+   export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem"
+   export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem"
+fi
+
+# Grab a connection string that only refers to *one* of the hosts in MONGODB_URI
+FIRST_HOST=$(node -p 'new (require("mongodb-connection-string-url").default)(process.env.MONGODB_URI).hosts[0]')
+# Use localhost:12345 as the URL for the single host that we connect to,
+# we configure the Socks5 proxy server script to redirect from this to FIRST_HOST
+export MONGODB_URI_SINGLEHOST="mongodb://localhost:12345/"
+
+# Compute path to socks5 fake server script in a way that works on Windows
+SOCKS5_SERVER_SCRIPT="$DRIVERS_TOOLS/.evergreen/socks5srv.py"
+if [ "Windows_NT" = "$OS" ]; then
+  SOCKS5_SERVER_SCRIPT=$(cygpath -w "$SOCKS5_SERVER_SCRIPT")
+fi
+
+# First, test with Socks5 + authentication required
+"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "localhost:12345 to $FIRST_HOST" &
+PID=$!
+env SOCKS5_CONFIG='["localhost",1080,"username","p4ssw0rd"]' npm run check:socks5
+[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
+  env MONGODB_URI='mongodb://localhost:12345/?proxyHost=localhost&proxyUsername=username&proxyPassword=p4ssw0rd' \
+  bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
+kill $PID
+
+# Second, test with Socks5 + no authentication
+"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "localhost:12345 to $FIRST_HOST" &
+PID=$!
+env SOCKS5_CONFIG='["localhost",1081]' npm run check:socks5
+[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
+  env MONGODB_URI='mongodb://localhost:12345/?proxyHost=localhost&proxyPort=1081' \
+  bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
+kill $PID
+
+# TODO: It might be worth using something more robust to control
+# the Socks5 proxy server script's lifetime

package-lock.json

@@ -34,7 +34,8 @@
   "dependencies": {
     "bson": "^4.6.0",
     "denque": "^2.0.1",
-    "mongodb-connection-string-url": "^2.3.2"
+    "mongodb-connection-string-url": "^2.3.2",
+    "socks": "^2.6.1"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -113,6 +114,7 @@
     "check:kerberos": "mocha --config \"test/manual/mocharc.json\" test/manual/kerberos.test.js",
     "check:tls": "mocha --config \"test/manual/mocharc.json\" test/manual/tls_support.test.js",
     "check:ldap": "mocha --config \"test/manual/mocharc.json\" test/manual/ldap.test.js",
+    "check:socks5": "mocha --config \"test/manual/mocharc.json\" test/manual/socks5.test.ts",
     "check:csfle": "mocha --file test/tools/runner test/integration/client-side-encryption",
     "check:snappy": "mocha --file test/tools/runner test/functional/unit_snappy.test.js",
     "prepare": "node etc/prepare.js",