From 012db39e4c9169eec37c4639e8b698c04a1a267b Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 17 Dec 2025 10:46:14 -0500 Subject: [PATCH 1/6] Modify model-validation tests so that they do not record duplicate properties. --- tests/functional/botocore/test_auth_config.py | 19 ++++-- tests/functional/botocore/test_endpoints.py | 3 +- .../botocore/test_socialmessaging.py | 4 +- tests/functional/test_no_event_streams.py | 61 +++++++++++-------- tests/functional/test_shadowing.py | 44 ++++++------- 5 files changed, 78 insertions(+), 53 deletions(-) diff --git a/tests/functional/botocore/test_auth_config.py b/tests/functional/botocore/test_auth_config.py index 4026b73ad43a..1222bd5f8149 100644 --- a/tests/functional/botocore/test_auth_config.py +++ b/tests/functional/botocore/test_auth_config.py @@ -62,7 +62,7 @@ def _all_test_cases(): # Skip service due to known mixed auth configurations. continue if auth_config: - auth_services.append([service, auth_config]) + auth_services.append([service, auth_config, service_model]) for operation in service_model.operation_names: operation_model = service_model.operation_model(operation) if operation_model.auth: @@ -74,8 +74,17 @@ def _all_test_cases(): @pytest.mark.validates_models -@pytest.mark.parametrize("auth_service, auth_config", AUTH_SERVICES) -def test_all_requirements_match_for_service(auth_service, auth_config): +@pytest.mark.parametrize( + "auth_service, auth_config, service_model", + AUTH_SERVICES +) +def test_all_requirements_match_for_service( + auth_service, + auth_config, + service_model, + record_property +): + record_property("aws_service", service_model.service_name) # Validates that all service-level signature types have the same requirements message = f'Found mixed signer requirements for service: {auth_service}' assert_all_requirements_match(auth_config, message) @@ -83,7 +92,9 @@ def test_all_requirements_match_for_service(auth_service, auth_config): @pytest.mark.validates_models @pytest.mark.parametrize("auth_service, operation_model", AUTH_OPERATIONS) -def test_all_requirements_match_for_operation(auth_service, operation_model): +def test_all_requirements_match_for_operation(auth_service, operation_model, record_property): + record_property("aws_service", operation_model.service_model.service_name) + record_property("aws_operation", operation_model.name) # Validates that all operation-level signature types have the same requirements message = f'Found mixed signer requirements for operation: {auth_service}.{operation_model.name}' auth_config = operation_model.auth diff --git a/tests/functional/botocore/test_endpoints.py b/tests/functional/botocore/test_endpoints.py index e6c582552eac..2408b47d2d2d 100644 --- a/tests/functional/botocore/test_endpoints.py +++ b/tests/functional/botocore/test_endpoints.py @@ -105,7 +105,7 @@ def _available_services(): @pytest.mark.validates_models @pytest.mark.parametrize("service_name", _available_services()) -def test_client_name_matches_hyphenized_service_id(service_name): +def test_client_name_matches_hyphenized_service_id(service_name, record_property): """Generates tests for each service to verify that the computed service named based on the service id matches the service name used to create a client (i.e the directory name in botocore/data) @@ -114,6 +114,7 @@ def test_client_name_matches_hyphenized_service_id(service_name): session = get_session() service_model = session.get_service_model(service_name) computed_name = service_model.service_id.replace(' ', '-').lower() + record_property('aws_service', service_name) # Handle known exceptions where we have renamed the service directory # for one reason or another. diff --git a/tests/functional/botocore/test_socialmessaging.py b/tests/functional/botocore/test_socialmessaging.py index c573fdee8809..b8171d82245c 100644 --- a/tests/functional/botocore/test_socialmessaging.py +++ b/tests/functional/botocore/test_socialmessaging.py @@ -34,9 +34,11 @@ def _get_all_xform_operations(): @pytest.mark.validates_models @pytest.mark.parametrize("operation, replacement", XFORM_OPERATIONS) -def test_known_replacements(operation, replacement): +def test_known_replacements(operation, replacement, record_property): # Validates that if a replacement shows up in the lowercased version of an # operation, we will keep all of those characters together in the final operation # name + record_property('aws_service', 'socialmessaging') + record_property('aws_operation', operation) assert replacement in xform_name(operation, '_') assert replacement in xform_name(operation, '-') diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index 1672d017ddfc..d3ca5a0e37fd 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -19,36 +19,47 @@ _ALLOWED_COMMANDS = ['s3api select-object-content'] -@pytest.mark.validates_models -def test_no_event_stream_unless_allowed(record_property): +def _generate_command_tests(): driver = create_clidriver() help_command = driver.create_help_command() - errors = [] - for command_name, command_obj in help_command.command_table.items(): + for command_name, command_obj in list(help_command.command_table.items()): sub_help = command_obj.create_help_command() if hasattr(sub_help, 'command_table'): for sub_name, sub_command in sub_help.command_table.items(): op_help = sub_command.create_help_command() model = op_help.obj if isinstance(model, OperationModel): - full_command = f'{command_name} {sub_name}' - if ( - model.has_event_stream_input - or model.has_event_stream_output - ): - if full_command in _ALLOWED_COMMANDS: - continue - # Store the service and operation in - # PyTest custom properties - record_property( - 'aws_service', model.service_model.service_name - ) - record_property('aws_operation', model.name) - supported_commands = '\n'.join(_ALLOWED_COMMANDS) - errors.append( - f'The {full_command} command uses event streams ' - 'which is only supported for these operations:\n' - f'{supported_commands}' - ) - if errors: - raise AssertionError('\n' + '\n'.join(errors)) + yield command_name, sub_name, sub_command, model + + +@pytest.mark.validates_models +@pytest.mark.parametrize( + "command_name, sub_name, sub_command, model", _generate_command_tests() +) +def test_no_event_stream_unless_allowed( + command_name, + sub_name, + sub_command, + model, + record_property +): + full_command = f'{command_name} {sub_name}' + if ( + ( + model.has_event_stream_input + or model.has_event_stream_output + ) + and full_command not in _ALLOWED_COMMANDS + ): + # Store the service and operation in + # PyTest custom properties + record_property( + 'aws_service', model.service_model.service_name + ) + record_property('aws_operation', model.name) + supported_commands = '\n'.join(_ALLOWED_COMMANDS) + assert full_command in _ALLOWED_COMMANDS, ( + f'The {full_command} command uses event streams ' + 'which is only supported for these operations:\n' + f'{supported_commands}' + ) diff --git a/tests/functional/test_shadowing.py b/tests/functional/test_shadowing.py index ffa3d8016053..8d5bcb1b428c 100644 --- a/tests/functional/test_shadowing.py +++ b/tests/functional/test_shadowing.py @@ -23,17 +23,18 @@ def _generate_command_tests(): for command_name, command_obj in list(help_command.command_table.items()): sub_help = command_obj.create_help_command() if hasattr(sub_help, 'command_table'): - yield command_name, sub_help.command_table, top_level_params + for sub_name, sub_command in sub_help.command_table.items(): + yield command_name, sub_name, sub_command, top_level_params @pytest.mark.validates_models @pytest.mark.parametrize( - "command_name, command_table, builtins", _generate_command_tests() + "command_name, sub_name, sub_command, builtins", _generate_command_tests() ) def test_no_shadowed_builtins( - command_name, command_table, builtins, record_property + command_name, sub_name, sub_command, builtins, record_property ): - """Verify no command params are shadowed or prefixed by the built in param. + """Verify no command params are shadowed or prefixed by the built-in param. The CLI parses all command line options into a single namespace. This means that option names must be unique and cannot conflict @@ -59,23 +60,22 @@ def test_no_shadowed_builtins( """ errors = [] - for sub_name, sub_command in command_table.items(): - op_help = sub_command.create_help_command() - model = op_help.obj - arg_table = op_help.arg_table - for arg_name in arg_table: - if any(p.startswith(arg_name) for p in builtins): - if isinstance(model, OperationModel): - # Store the service and operation in - # PyTest custom properties - record_property( - 'aws_service', model.service_model.service_name - ) - record_property('aws_operation', model.name) - # Then we are shadowing or prefixing a top level argument - errors.append( - 'Shadowing/Prefixing a top level option: ' - f'{command_name}.{sub_name}.{arg_name}' - ) + op_help = sub_command.create_help_command() + model = op_help.obj + arg_table = op_help.arg_table + for arg_name in arg_table: + if any(p.startswith(arg_name) for p in builtins): + # Then we are shadowing or prefixing a top level argument + errors.append( + 'Shadowing/Prefixing a top level option: ' + f'{command_name}.{sub_name}.{arg_name}' + ) if errors: + if isinstance(model, OperationModel): + # Store the service and operation in + # PyTest custom properties + record_property( + 'aws_service', model.service_model.service_name + ) + record_property('aws_operation', model.name) raise AssertionError('\n' + '\n'.join(errors)) From 4b316fee41f794f7beaccacb4661075f2f25a32f Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 17 Dec 2025 10:51:41 -0500 Subject: [PATCH 2/6] Remove accidental use of list. --- tests/functional/test_no_event_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index d3ca5a0e37fd..9c366f3666dd 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -22,7 +22,7 @@ def _generate_command_tests(): driver = create_clidriver() help_command = driver.create_help_command() - for command_name, command_obj in list(help_command.command_table.items()): + for command_name, command_obj in help_command.command_table.items(): sub_help = command_obj.create_help_command() if hasattr(sub_help, 'command_table'): for sub_name, sub_command in sub_help.command_table.items(): From 19494b3f1517f105e256e4ed47d68516edd83318 Mon Sep 17 00:00:00 2001 From: aemous Date: Wed, 17 Dec 2025 10:53:16 -0500 Subject: [PATCH 3/6] Fix if-statement logic bug. --- tests/functional/test_no_event_streams.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index 9c366f3666dd..70d7a28ed469 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -44,13 +44,7 @@ def test_no_event_stream_unless_allowed( record_property ): full_command = f'{command_name} {sub_name}' - if ( - ( - model.has_event_stream_input - or model.has_event_stream_output - ) - and full_command not in _ALLOWED_COMMANDS - ): + if model.has_event_stream_input or model.has_event_stream_output: # Store the service and operation in # PyTest custom properties record_property( From 0516de50fa022e68afd497b1bc5d3a3563508f41 Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 18 Dec 2025 12:31:34 -0500 Subject: [PATCH 4/6] Modify tests to allocate less memory than needed. --- tests/functional/botocore/test_auth_config.py | 7 +++---- tests/functional/test_no_event_streams.py | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/functional/botocore/test_auth_config.py b/tests/functional/botocore/test_auth_config.py index 1222bd5f8149..ad60fb416e4a 100644 --- a/tests/functional/botocore/test_auth_config.py +++ b/tests/functional/botocore/test_auth_config.py @@ -62,7 +62,7 @@ def _all_test_cases(): # Skip service due to known mixed auth configurations. continue if auth_config: - auth_services.append([service, auth_config, service_model]) + auth_services.append([service, auth_config]) for operation in service_model.operation_names: operation_model = service_model.operation_model(operation) if operation_model.auth: @@ -75,16 +75,15 @@ def _all_test_cases(): @pytest.mark.validates_models @pytest.mark.parametrize( - "auth_service, auth_config, service_model", + "auth_service, auth_config", AUTH_SERVICES ) def test_all_requirements_match_for_service( auth_service, auth_config, - service_model, record_property ): - record_property("aws_service", service_model.service_name) + record_property("aws_service", auth_service) # Validates that all service-level signature types have the same requirements message = f'Found mixed signer requirements for service: {auth_service}' assert_all_requirements_match(auth_config, message) diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index 70d7a28ed469..822c3fe53ec3 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -29,17 +29,16 @@ def _generate_command_tests(): op_help = sub_command.create_help_command() model = op_help.obj if isinstance(model, OperationModel): - yield command_name, sub_name, sub_command, model + yield command_name, sub_name, model @pytest.mark.validates_models @pytest.mark.parametrize( - "command_name, sub_name, sub_command, model", _generate_command_tests() + "command_name, sub_name, model", _generate_command_tests() ) def test_no_event_stream_unless_allowed( command_name, sub_name, - sub_command, model, record_property ): From e4fe854336e65584ad3dd2be2842007eca2dc2dd Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 18 Dec 2025 13:13:28 -0500 Subject: [PATCH 5/6] Modify test_no_event_stream parametrization to use less memory. --- tests/functional/test_no_event_streams.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index 822c3fe53ec3..8c04d3107192 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -28,8 +28,17 @@ def _generate_command_tests(): for sub_name, sub_command in sub_help.command_table.items(): op_help = sub_command.create_help_command() model = op_help.obj + # Extract the properties needed for tests to avoid + # parametrizing entire model objects, which may cause + # excessive memory usage. + model_description = { + 'has_event_stream_input': model.has_event_stream_input, + 'has_event_stream_output': model.has_event_stream_output, + 'service_name': model.service_model.service_name, + 'name': model.name, + } if isinstance(model, OperationModel): - yield command_name, sub_name, model + yield command_name, sub_name, model_description @pytest.mark.validates_models @@ -43,13 +52,13 @@ def test_no_event_stream_unless_allowed( record_property ): full_command = f'{command_name} {sub_name}' - if model.has_event_stream_input or model.has_event_stream_output: + if model['has_event_stream_input'] or model['has_event_stream_output']: # Store the service and operation in # PyTest custom properties record_property( - 'aws_service', model.service_model.service_name + 'aws_service', model['service_name'] ) - record_property('aws_operation', model.name) + record_property('aws_operation', model['name']) supported_commands = '\n'.join(_ALLOWED_COMMANDS) assert full_command in _ALLOWED_COMMANDS, ( f'The {full_command} command uses event streams ' From 6d8d713cabdd73e305dde85999688d70f3375d6b Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 18 Dec 2025 13:34:54 -0500 Subject: [PATCH 6/6] Fix bug with accessing attribute before validating object has the correct type. --- tests/functional/test_no_event_streams.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/functional/test_no_event_streams.py b/tests/functional/test_no_event_streams.py index 8c04d3107192..d9bb2e90a47f 100644 --- a/tests/functional/test_no_event_streams.py +++ b/tests/functional/test_no_event_streams.py @@ -28,16 +28,16 @@ def _generate_command_tests(): for sub_name, sub_command in sub_help.command_table.items(): op_help = sub_command.create_help_command() model = op_help.obj - # Extract the properties needed for tests to avoid - # parametrizing entire model objects, which may cause - # excessive memory usage. - model_description = { - 'has_event_stream_input': model.has_event_stream_input, - 'has_event_stream_output': model.has_event_stream_output, - 'service_name': model.service_model.service_name, - 'name': model.name, - } if isinstance(model, OperationModel): + # Extract the properties needed for tests to avoid + # parametrizing entire model objects, which may cause + # excessive memory usage. + model_description = { + 'has_event_stream_input': model.has_event_stream_input, + 'has_event_stream_output': model.has_event_stream_output, + 'service_name': model.service_model.service_name, + 'name': model.name, + } yield command_name, sub_name, model_description