From 764469693514043eb4d053c913a8bf6992f17497 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 10:12:25 -0500 Subject: [PATCH 1/8] PYTHON-5217 Update Atlas Data Lake tests --- .evergreen/config.yml | 21 --------------------- .evergreen/scripts/configure-env.sh | 2 ++ .evergreen/scripts/generate_config.py | 10 +++++++++- .evergreen/scripts/setup_tests.py | 5 +++++ .evergreen/scripts/teardown_tests.py | 4 ++++ 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ee29d65f2b..15bdac315d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -213,20 +213,6 @@ functions: params: file: ${DRIVERS_TOOLS}/mo-expansion.yml - "bootstrap data lake": - - command: subprocess.exec - type: setup - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - - command: subprocess.exec - type: setup - params: - binary: bash - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - "run doctests": - command: subprocess.exec type: test @@ -404,13 +390,6 @@ tasks: TOPOLOGY: "replica_set" - func: "run tests" - - name: atlas-data-lake-tests - commands: - - func: "bootstrap data lake" - - func: "run tests" - vars: - TEST_NAME: "data_lake" - - name: "test-aws-lambda-deployed" commands: - command: ec2.assume_role diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fa37b8fb08..fb09937c86 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -77,7 +77,9 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS BRANCH=master +BRANCH=PYTHON-5217 ORG=mongodb-labs +ORG=blink1073 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index b9f9377066..50f6cdb67c 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -601,7 +601,7 @@ def create_atlas_data_lake_variants(): variants = [] host = HOSTS["ubuntu22"] for python, c_ext in product(MIN_MAX_PYTHON, C_EXTS): - tasks = ["atlas-data-lake-tests"] + tasks = [".atlas_data_lake"] expansions = dict(AUTH="auth") handle_c_ext(c_ext, expansions) display_name = get_display_name("Atlas Data Lake", host, python=python, **expansions) @@ -970,6 +970,14 @@ def create_perf_tasks(): return tasks +def create_atlas_data_lake_tasks(): + vars = dict(TEST_NAME="data_lake") + test_func = FunctionCall(func="run tests", vars=vars) + task_name = "test-atlas-data-lake" + tags = ["atlas_data_lake"] + return [EvgTask(name=task_name, tags=tags, commands=[test_func])] + + def create_ocsp_tasks(): tasks = [] tests = [ diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 6d7c8037c7..b3af606363 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -175,6 +175,11 @@ def handle_test_env() -> None: if not config: AUTH = "noauth" + if test_name == "data_lake": + run_command(f"{DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") + run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") + AUTH = "auth" + if AUTH != "noauth": if test_name == "data_lake": config = read_env(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/secrets-export.sh") diff --git a/.evergreen/scripts/teardown_tests.py b/.evergreen/scripts/teardown_tests.py index b081478423..d27cb8682e 100644 --- a/.evergreen/scripts/teardown_tests.py +++ b/.evergreen/scripts/teardown_tests.py @@ -57,6 +57,10 @@ teardown_mod_wsgi() +# Tear down data_lake if applicable. +elif TEST_NAME == "data_lake": + run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/teardown.sh") + # Tear down coverage if applicable. if os.environ.get("COVERAGE"): shutil.rmtree(".pytest_cache", ignore_errors=True) From cd8c6a79acb9ca85e5a69f40ae9a167d80f256a4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 10:12:46 -0500 Subject: [PATCH 2/8] update tasks --- .evergreen/generated_configs/tasks.yml | 8 ++++++++ .evergreen/generated_configs/variants.yml | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 5b5cf92d68..6a23b7ac8d 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -8,6 +8,14 @@ tasks: TEST_NAME: atlas_connect tags: [atlas_connect] + # Atlas data lake tests + - name: test-atlas-data-lake + commands: + - func: run tests + vars: + TEST_NAME: data_lake + tags: [atlas_data_lake] + # Aws tests - name: test-auth-aws-4.4-regular commands: diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 864b061a15..572b95f621 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -67,7 +67,7 @@ buildvariants: # Atlas data lake tests - name: atlas-data-lake-ubuntu-22-python3.9-auth-no-c tasks: - - name: atlas-data-lake-tests + - name: .atlas_data_lake display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth No C run_on: - ubuntu2204-small @@ -77,7 +77,7 @@ buildvariants: PYTHON_BINARY: /opt/python/3.9/bin/python3 - name: atlas-data-lake-ubuntu-22-python3.9-auth tasks: - - name: atlas-data-lake-tests + - name: .atlas_data_lake display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth run_on: - ubuntu2204-small @@ -86,7 +86,7 @@ buildvariants: PYTHON_BINARY: /opt/python/3.9/bin/python3 - name: atlas-data-lake-ubuntu-22-python3.13-auth-no-c tasks: - - name: atlas-data-lake-tests + - name: .atlas_data_lake display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth No C run_on: - ubuntu2204-small @@ -96,7 +96,7 @@ buildvariants: PYTHON_BINARY: /opt/python/3.13/bin/python3 - name: atlas-data-lake-ubuntu-22-python3.13-auth tasks: - - name: atlas-data-lake-tests + - name: .atlas_data_lake display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth run_on: - ubuntu2204-small From dca6fdbf8fb08330545f28ecc3b7547e60f9f636 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 10:16:11 -0500 Subject: [PATCH 3/8] fix command --- .evergreen/scripts/setup_tests.py | 4 ++-- CONTRIBUTING.md | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index b3af606363..9c41b30f41 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -176,8 +176,8 @@ def handle_test_env() -> None: AUTH = "noauth" if test_name == "data_lake": - run_command(f"{DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") - run_command(f"{DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") + run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") AUTH = "auth" if AUTH != "noauth": diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47eb01dbf0..e466c6847e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -286,16 +286,23 @@ Note: these tests can only be run from an Evergreen Linux host that has the Pyth The `mode` can be `standalone` or `embedded`. For the `replica_set` version of the tests, use `TOPOLOGY=replica_set just run-server`. +### Atlas Data Lake tests. + +You must have `docker` or `podman` installed locally. + +- Run `just setup-tests data_lake`. +- Run `just run-tests`. + ### OCSP tests - - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. - This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. - MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. - - Run `just run-server ocsp`. - - Run `just setup-tests ocsp ` (options are "valid", "revoked", "valid-delegate", "revoked-delegate"). - - Run `just run-tests` +- Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. +This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. +- Run `just run-server ocsp`. +- Run `just setup-tests ocsp ` (options are "valid", "revoked", "valid-delegate", "revoked-delegate"). +- Run `just run-tests` - If you are running one of the `no-responder` tests, omit the `run-server` step. +If you are running one of the `no-responder` tests, omit the `run-server` step. ### Perf Tests From 46b5050d4cf67ac3aeb7925e1100d2dd7eefad2b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 10:20:31 -0500 Subject: [PATCH 4/8] clean up tasks --- .evergreen/generated_configs/tasks.yml | 6 +++-- .evergreen/generated_configs/variants.yml | 30 +++-------------------- .evergreen/scripts/generate_config.py | 22 ++++++++--------- 3 files changed, 19 insertions(+), 39 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 6a23b7ac8d..a5a09cefa4 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -12,8 +12,10 @@ tasks: - name: test-atlas-data-lake commands: - func: run tests - vars: - TEST_NAME: data_lake + tags: [atlas_data_lake] + - name: test-atlas-data-lake + commands: + - func: run tests tags: [atlas_data_lake] # Aws tests diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 572b95f621..78f181fc64 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -65,43 +65,21 @@ buildvariants: PYTHON_BINARY: /opt/python/3.13/bin/python3 # Atlas data lake tests - - name: atlas-data-lake-ubuntu-22-python3.9-auth-no-c + - name: atlas-data-lake-ubuntu-22-python3.9 tasks: - name: .atlas_data_lake - display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth No C + display_name: Atlas Data Lake Ubuntu-22 Python3.9 run_on: - ubuntu2204-small expansions: - AUTH: auth - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.9-auth - tasks: - - name: .atlas_data_lake - display_name: Atlas Data Lake Ubuntu-22 Python3.9 Auth - run_on: - - ubuntu2204-small - expansions: - AUTH: auth PYTHON_BINARY: /opt/python/3.9/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.13-auth-no-c + - name: atlas-data-lake-ubuntu-22-python3.13 tasks: - name: .atlas_data_lake - display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth No C + display_name: Atlas Data Lake Ubuntu-22 Python3.13 run_on: - ubuntu2204-small expansions: - AUTH: auth - NO_EXT: "1" - PYTHON_BINARY: /opt/python/3.13/bin/python3 - - name: atlas-data-lake-ubuntu-22-python3.13-auth - tasks: - - name: .atlas_data_lake - display_name: Atlas Data Lake Ubuntu-22 Python3.13 Auth - run_on: - - ubuntu2204-small - expansions: - AUTH: auth PYTHON_BINARY: /opt/python/3.13/bin/python3 # Aws auth tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 50f6cdb67c..22129a8be2 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -600,14 +600,10 @@ def create_no_c_ext_variants(): def create_atlas_data_lake_variants(): variants = [] host = HOSTS["ubuntu22"] - for python, c_ext in product(MIN_MAX_PYTHON, C_EXTS): + for python in MIN_MAX_PYTHON: tasks = [".atlas_data_lake"] - expansions = dict(AUTH="auth") - handle_c_ext(c_ext, expansions) - display_name = get_display_name("Atlas Data Lake", host, python=python, **expansions) - variant = create_variant( - tasks, display_name, host=host, python=python, expansions=expansions - ) + display_name = get_display_name("Atlas Data Lake", host, python=python) + variant = create_variant(tasks, display_name, host=host, python=python) variants.append(variant) return variants @@ -971,11 +967,15 @@ def create_perf_tasks(): def create_atlas_data_lake_tasks(): - vars = dict(TEST_NAME="data_lake") - test_func = FunctionCall(func="run tests", vars=vars) - task_name = "test-atlas-data-lake" tags = ["atlas_data_lake"] - return [EvgTask(name=task_name, tags=tags, commands=[test_func])] + tasks = [] + for c_ext in C_EXTS: + vars = dict(TEST_NAME="data_lake") + vars = handle_c_ext(c_ext, vars) + test_func = FunctionCall(func="run tests", vars=vars) + task_name = "test-atlas-data-lake" + tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) + return tasks def create_ocsp_tasks(): From 7520e21129e2039cdeea43f1172c54b5016b669d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 10:21:11 -0500 Subject: [PATCH 5/8] clean up tasks --- .evergreen/generated_configs/tasks.yml | 4 ++-- .evergreen/scripts/generate_config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index a5a09cefa4..d1ebe27593 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -9,11 +9,11 @@ tasks: tags: [atlas_connect] # Atlas data lake tests - - name: test-atlas-data-lake + - name: test-atlas-data-lake-with_ext commands: - func: run tests tags: [atlas_data_lake] - - name: test-atlas-data-lake + - name: test-atlas-data-lake-without_ext commands: - func: run tests tags: [atlas_data_lake] diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 22129a8be2..0f5943a470 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -973,7 +973,7 @@ def create_atlas_data_lake_tasks(): vars = dict(TEST_NAME="data_lake") vars = handle_c_ext(c_ext, vars) test_func = FunctionCall(func="run tests", vars=vars) - task_name = "test-atlas-data-lake" + task_name = f"test-atlas-data-lake-{c_ext}" tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) return tasks From 83122bc6ef87eeb2d2516e2901ea64ab1aaf2c99 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 11:29:58 -0500 Subject: [PATCH 6/8] use upstream --- .evergreen/scripts/configure-env.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fb09937c86..fa37b8fb08 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -77,9 +77,7 @@ EOT # Write the .env file for drivers-tools. rm -rf $DRIVERS_TOOLS BRANCH=master -BRANCH=PYTHON-5217 ORG=mongodb-labs -ORG=blink1073 git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS cat < ${DRIVERS_TOOLS}/.env From 46986b66ae6650108f4ad6e764bc4ecef0b1dfcf Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 11:39:09 -0500 Subject: [PATCH 7/8] fix tasks --- .evergreen/generated_configs/tasks.yml | 5 +++++ .evergreen/scripts/generate_config.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index d1ebe27593..7d03822a6f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -12,10 +12,15 @@ tasks: - name: test-atlas-data-lake-with_ext commands: - func: run tests + vars: + TEST_NAME: data_lake + NO_EXT: "1" tags: [atlas_data_lake] - name: test-atlas-data-lake-without_ext commands: - func: run tests + vars: + TEST_NAME: data_lake tags: [atlas_data_lake] # Aws tests diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 0f5943a470..ebb4b5d31c 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -213,7 +213,7 @@ def zip_cycle(*iterables, empty_default=None): yield tuple(next(i, empty_default) for i in cycles) -def handle_c_ext(c_ext, expansions): +def handle_c_ext(c_ext, expansions) -> None: """Handle c extension option.""" if c_ext == C_EXTS[0]: expansions["NO_EXT"] = "1" @@ -971,7 +971,7 @@ def create_atlas_data_lake_tasks(): tasks = [] for c_ext in C_EXTS: vars = dict(TEST_NAME="data_lake") - vars = handle_c_ext(c_ext, vars) + handle_c_ext(c_ext, vars) test_func = FunctionCall(func="run tests", vars=vars) task_name = f"test-atlas-data-lake-{c_ext}" tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) From 71e031af95e1a83d4b82db8e2d8034c0329f5b22 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 18 Mar 2025 11:42:25 -0500 Subject: [PATCH 8/8] add comment --- .evergreen/scripts/setup_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 9c41b30f41..56011341a3 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -176,6 +176,7 @@ def handle_test_env() -> None: AUTH = "noauth" if test_name == "data_lake": + # Stop any running mongo-orchestration which might be using the port. run_command(f"bash {DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh") run_command(f"bash {DRIVERS_TOOLS}/.evergreen/atlas_data_lake/setup.sh") AUTH = "auth"