diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index c1ebcdb..2c517f4 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -7,7 +7,7 @@ from apiflask import APIBlueprint, Schema from apiflask.fields import String, Boolean from marshmallow.fields import Nested -from flask import Response +from flask import Response, current_app from app.services.validation_service import ( queue_ro_crate_validation_task, @@ -81,7 +81,10 @@ def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: else: profile_name = None - return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) + profiles_path = current_app.config["PROFILES_PATH"] + + return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, + webhook_url, profiles_path) @post_routes_bp.post("/validate_metadata") diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 67dde94..b51a088 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -25,7 +25,8 @@ def queue_ro_crate_validation_task( - minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None + minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None, + profiles_path=None ) -> tuple[Response, int]: """ Queues an RO-Crate for validation with Celery. @@ -51,7 +52,8 @@ def queue_ro_crate_validation_task( raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) try: - process_validation_task_by_id.delay(minio_config, crate_id, root_path, profile_name, webhook_url) + process_validation_task_by_id.delay(minio_config, crate_id, root_path, + profile_name, webhook_url, profiles_path) return jsonify({"message": "Validation in progress"}), 202 except Exception as e: diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index 0a62b55..3b178e5 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -29,7 +29,8 @@ @celery.task def process_validation_task_by_id( - minio_config: dict, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None + minio_config: dict, crate_id: str, root_path: str, profile_name: str | None, + webhook_url: str | None, profiles_path: str | None ) -> None: """ Background task to process the RO-Crate validation by ID. @@ -56,7 +57,7 @@ def process_validation_task_by_id( logging.info(f"Processing validation task for {file_path}") # Perform validation: - validation_result = perform_ro_crate_validation(file_path, profile_name) + validation_result = perform_ro_crate_validation(file_path, profile_name, profiles_path=profiles_path) if isinstance(validation_result, str): logging.error(f"Validation failed: {validation_result}") @@ -158,7 +159,7 @@ def process_validation_task_by_metadata( def perform_ro_crate_validation( - file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None + file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None, profiles_path: Optional[str] = None ) -> ValidationResult | str: """ Validates an RO-Crate using the provided file path and profile name. @@ -166,6 +167,7 @@ def perform_ro_crate_validation( :param file_path: The path to the RO-Crate file to validate :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will attempt to determine the profile. + :param profiles_path: The path to the profiles definition directory :param skip_checks_list: A list of checks to skip, if needed :return: The validation result. :raises Exception: If an error occurs during the validation process. @@ -183,7 +185,8 @@ def perform_ro_crate_validation( settings = services.ValidationSettings( rocrate_uri=full_file_path, **({"profile_identifier": profile_name} if profile_name else {}), - **({"skip_checks": skip_checks_list} if skip_checks_list else {}) + **({"skip_checks": skip_checks_list} if skip_checks_list else {}), + **({"extra_profiles_path": profiles_path} if profiles_path else {}) ) return services.validate(settings) diff --git a/app/utils/config.py b/app/utils/config.py index a57b63f..28e71ae 100644 --- a/app/utils/config.py +++ b/app/utils/config.py @@ -10,34 +10,32 @@ from flask import Flask +def get_env(name: str, default=None, required=False): + value = os.environ.get(name, default) + if required and value is None: + raise RuntimeError(f"Missing required environment variable: {name}") + return value + + class Config: """Base configuration class for the Flask application.""" - SECRET_KEY = os.getenv("SECRET_KEY", "my_precious") - # Celery configuration: - CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL") - CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND") + CELERY_BROKER_URL = get_env("CELERY_BROKER_URL", required=False) + CELERY_RESULT_BACKEND = get_env("CELERY_RESULT_BACKEND", required=False) - # MinIO configuration: - MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT") - MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY") - MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY") - MINIO_BUCKET_NAME = os.getenv("MINIO_BUCKET_NAME", "bucket-name") + # rocrate validator configuration: + PROFILES_PATH = get_env("PROFILES_PATH", required=False) class DevelopmentConfig(Config): """Development configuration class.""" - DEBUG = True - ENV = "development" class ProductionConfig(Config): """Production configuration class.""" - DEBUG = False - ENV = "production" class InvalidAPIUsage(Exception): @@ -63,10 +61,13 @@ def make_celery(app: Flask = None) -> Celery: :param app: The Flask application to use. :return: The Celery instance. """ + env = os.environ.get("FLASK_ENV", "development") + config_cls = ProductionConfig if env == "production" else DevelopmentConfig + celery = Celery( app.import_name if app else __name__, - broker=os.getenv("CELERY_BROKER_URL"), - backend=os.getenv("CELERY_RESULT_BACKEND"), + broker=config_cls.CELERY_BROKER_URL, + backend=config_cls.CELERY_RESULT_BACKEND, ) if app: diff --git a/docker-compose-develop.yml b/docker-compose-develop.yml index fa7661d..e6b6a0e 100644 --- a/docker-compose-develop.yml +++ b/docker-compose-develop.yml @@ -32,9 +32,12 @@ services: - MINIO_ROOT_USER=${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME} + - PROFILES_PATH=/app/profiles depends_on: - redis - minio + volumes: + - ./tests/data/rocrate_validator_profiles:/app/profiles:ro redis: image: "redis:alpine" diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index f527501..e50b511 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -13,7 +13,7 @@ def client(): # Test POST API: /v1/ro_crates/{crate_id}/validation @pytest.mark.parametrize( - "crate_id, payload, status_code, response_json", + "crate_id, payload, profiles_path, status_code, response_json", [ ( "crate-123", { @@ -27,7 +27,9 @@ def client(): "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ( "crate-123", { @@ -38,9 +40,11 @@ def client(): "ssl": False, "bucket": "test_bucket" }, - "root_path": "base_path", + "root_path": "base_path", "webhook_url": "https://webhook.example.com", - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ( "crate-123", { @@ -51,9 +55,11 @@ def client(): "ssl": False, "bucket": "test_bucket" }, - "root_path": "base_path", + "root_path": "base_path", "profile_name": "default" - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ( "crate-123", { @@ -66,7 +72,9 @@ def client(): }, "webhook_url": "https://webhook.example.com", "profile_name": "default" - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ( "crate-123", { @@ -77,14 +85,17 @@ def client(): "ssl": False, "bucket": "test_bucket" }, - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ], ids=["validate_by_id", "validate_with_missing_profile_name", "validate_with_missing_webhook_url", "validate_with_missing_root_path", "validate_with_missing_root_path_and_profile_name_and_webhook_url"] ) -def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dict, status_code: int, response_json: dict): +def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dict, + profiles_path: str, status_code: int, response_json: dict): with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: mock_queue.return_value = (response_json, status_code) @@ -96,7 +107,7 @@ def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dic webhook_url = payload["webhook_url"] if "webhook_url" in payload else None assert response.status_code == status_code assert response.json == response_json - mock_queue.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) + mock_queue.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url, profiles_path) @pytest.mark.parametrize( diff --git a/tests/test_integration.py b/tests/test_integration.py index 1e90a1a..63941c4 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -350,6 +350,80 @@ def test_directory_rocrate_validation(): assert response_result["passed"] is False +def test_extra_profile_rocrate_validation(): + ro_crate = "ro_crate_2" + profile_name = "alpha-crate-0.1" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + post_payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + }, + "profile_name": profile_name + } + get_payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # POST action and tests + response = requests.post(url_post, json=post_payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=get_payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=get_payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + def test_ignore_rocrates_not_on_basepath(): ro_crate = "ro_crate_4" url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" diff --git a/tests/test_services.py b/tests/test_services.py index c7d50c3..ccebeba 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -22,7 +22,7 @@ def flask_app(): # Test function: queue_ro_crate_validation_task @pytest.mark.parametrize( - "crate_id, rocrate_exists, minio_client, delay_side_effects, payload, status_code, response_dict", + "crate_id, rocrate_exists, minio_client, delay_side_effects, payload, profiles_path, status_code, response_dict", [ ( "crate123", True, "minio_client", None, @@ -37,7 +37,9 @@ def flask_app(): "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" - }, 202, {"message": "Validation in progress"} + }, + None, + 202, {"message": "Validation in progress"} ), ( "crate123", True, "minio_client", Exception("Celery down"), @@ -52,7 +54,9 @@ def flask_app(): "root_path": "base_path", "webhook_url": "https://webhook.example.com", "profile_name": "default" - }, 500, {"error": "Celery down"} + }, + None, + 500, {"error": "Celery down"} ), ], ids=["successful_queue", "celery_server_down"] @@ -65,7 +69,7 @@ def test_queue_ro_crate_validation_task( mock_exists, mock_delay, flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, minio_client: str, - delay_side_effects: Exception, payload: dict, status_code: int, response_dict: dict + delay_side_effects: Exception, payload: dict, profiles_path: str, status_code: int, response_dict: dict ): mock_delay.side_effect = delay_side_effects mock_exists.return_value = rocrate_exists @@ -76,11 +80,12 @@ def test_queue_ro_crate_validation_task( profile_name = payload["profile_name"] if "profile_name" in payload else None webhook_url = payload["webhook_url"] if "webhook_url" in payload else None - response, status_code = queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) + response, status_code = queue_ro_crate_validation_task(minio_config, crate_id, root_path, + profile_name, webhook_url, profiles_path) mock_client.assert_called_once_with(minio_config) mock_exists.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, root_path) - mock_delay.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) + mock_delay.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url, profiles_path) assert status_code == status_code assert response.json == response_dict diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py index afa11c2..49c3fed 100644 --- a/tests/test_validation_tasks.py +++ b/tests/test_validation_tasks.py @@ -17,7 +17,7 @@ @pytest.mark.parametrize( "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, " + - "return_value, webhook, profile, val_success, val_result, minio_client", + "return_value, webhook, profile, profiles_path, val_success, val_result, minio_client", [ ( { @@ -28,7 +28,7 @@ "bucket": "test_bucket" }, "crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "https://example.com/hook", "profileA", None, True, '{"status": "valid"}', "minio_client" ), ( @@ -40,7 +40,7 @@ "bucket": "test_bucket" }, "crate123", True, False, True, "/tmp/crate123", - "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "https://example.com/hook", "profileA", None, True, '{"status": "valid"}', "minio_client" ), ( @@ -52,7 +52,7 @@ "bucket": "test_bucket" }, "crate123", True, False, True, "/tmp/crate123", - None, "profileA", True, '{"status": "valid"}', + None, "profileA", None, True, '{"status": "valid"}', "minio_client" ), ], @@ -80,7 +80,7 @@ def test_process_validation( mock_rmtree, mock_client, minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, - return_value: str, webhook: str, profile: str, val_success: bool, val_result: str, minio_client: str + return_value: str, webhook: str, profile: str, profiles_path: str, val_success: bool, val_result: str, minio_client: str ): mock_exists.return_value = os_path_exists mock_isfile.return_value = os_path_isfile @@ -93,11 +93,11 @@ def test_process_validation( mock_validation_result.to_json.return_value = val_result mock_validate.return_value = mock_validation_result - process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook, profiles_path) mock_client.assert_called_once_with(minio_config) mock_fetch.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "") - mock_validate.assert_called_once_with(return_value, profile) + mock_validate.assert_called_once_with(return_value, profile, profiles_path=profiles_path) mock_update.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "", val_result) if webhook is not None: mock_webhook.assert_called_once_with(webhook, val_result) @@ -113,7 +113,7 @@ def test_process_validation( @pytest.mark.parametrize( "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_fetch, " - + "webhook, profile, return_validate, validate_side_effect, fetch_side_effect, minio_client", + + "webhook, profile, profiles_path, return_validate, validate_side_effect, fetch_side_effect, minio_client", [ ( { @@ -124,7 +124,7 @@ def test_process_validation( "bucket": "test_bucket" }, "crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", "Validation failed", None, None, + "https://example.com/hook", "profileA", None, "Validation failed", None, None, "minio_client" ), ( @@ -136,7 +136,7 @@ def test_process_validation( "bucket": "test_bucket" }, "crate123", True, True, False, "/tmp/crate.zip", - "https://example.com/hook", "profileA", None, Exception("Unexpected error"), None, + "https://example.com/hook", "profileA", None, None, Exception("Unexpected error"), None, "minio_client" ), ( @@ -148,7 +148,7 @@ def test_process_validation( "bucket": "test_bucket" }, "crate123", False, False, False, None, - "https://example.com/hook", "profileA", None, None, Exception("MinIO fetch failed"), + "https://example.com/hook", "profileA", None, None, None, Exception("MinIO fetch failed"), "minio_client" ), ], @@ -177,7 +177,7 @@ def test_process_validation_failure( mock_rmtree, mock_client, minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, - return_fetch: str, webhook: str, profile: str, return_validate: str, + return_fetch: str, webhook: str, profile: str, profiles_path: str, return_validate: str, validate_side_effect: Exception, fetch_side_effect: Exception, minio_client: str ): mock_exists.return_value = os_path_exists @@ -195,10 +195,10 @@ def test_process_validation_failure( else: mock_validate.side_effect = validate_side_effect - process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook, profiles_path) if fetch_side_effect is None: - mock_validate.assert_called_once_with(return_fetch, profile) + mock_validate.assert_called_once_with(return_fetch, profile, profiles_path=profiles_path) else: mock_validate.assert_not_called()