diff --git a/config.docker.yml b/config.docker.yml index aa7f751d..177df45c 100644 --- a/config.docker.yml +++ b/config.docker.yml @@ -18,13 +18,14 @@ auth: # Keycloak Auth # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider - # server_url: http://keycloak:8080 - # realm_name: manually_created - # client_id: manually_created - # client_secret: generated_by_keycloak - # redirect_uri: http://localhost:3000/auth/callback - # scope: email - # verify_ssl: False + # keycloak: + # server_url: http://keycloak:8080 + # realm_name: manually_created + # client_id: manually_created + # client_secret: generated_by_keycloak + # redirect_uri: http://localhost:3000/auth/callback + # scope: email + # verify_ssl: False ui: @@ -51,7 +52,6 @@ server: expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials] - scheduler: transfer_fetching_timeout_seconds: 200 diff --git a/config.yml b/config.yml index 259f57e1..c3deb0b9 100644 --- a/config.yml +++ b/config.yml @@ -18,13 +18,14 @@ auth: # Keycloak Auth # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider - # server_url: http://localhost:8080 - # realm_name: manually_created - # client_id: manually_created - # client_secret: generated_by_keycloak - # redirect_uri: http://localhost:3000/auth/callback - # scope: email - # verify_ssl: False + # keycloak: + # server_url: http://localhost:8080 + # realm_name: manually_created + # client_id: manually_created + # client_secret: generated_by_keycloak + # redirect_uri: http://localhost:3000/auth/callback + # scope: email + # verify_ssl: False ui: diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py index bdc8cf38..6aac1a7c 100644 --- a/syncmaster/db/repositories/utils.py +++ b/syncmaster/db/repositories/utils.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING from cryptography.fernet import Fernet -from pydantic import SecretStr if TYPE_CHECKING: from syncmaster.scheduler.settings import SchedulerAppSettings @@ -23,11 +22,6 @@ def decrypt_auth_data( return json.loads(decrypted) -def _json_default(value): - if isinstance(value, SecretStr): - return value.get_secret_value() - - def encrypt_auth_data( value: dict, settings: WorkerAppSettings | SchedulerAppSettings | ServerAppSettings, @@ -37,7 +31,6 @@ def encrypt_auth_data( value, ensure_ascii=False, sort_keys=True, - default=_json_default, ) encrypted = encryptor.encrypt(serialized.encode("utf-8")) return encrypted.decode("utf-8") diff --git a/syncmaster/schemas/v1/auth/basic.py b/syncmaster/schemas/v1/auth/basic.py index 8bf84c75..6c770438 100644 --- a/syncmaster/schemas/v1/auth/basic.py +++ b/syncmaster/schemas/v1/auth/basic.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadBasicAuthSchema(BaseModel): + +class ReadBasicAuthSchema(SecretDumpMixin): type: Literal["basic"] = Field(description="Auth type") user: str diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py index 9b03bd6d..81f0d1a2 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/bearer.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadIcebergRESTCatalogBearerAuthSchema(BaseModel): + +class ReadIcebergRESTCatalogBearerAuthSchema(SecretDumpMixin): type: Literal["iceberg_rest_bearer"] = Field(description="Auth type") diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py index bbbd72bd..024c41da 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_delegated/oauth2_client_credentials.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin from syncmaster.schemas.v1.types import URL -class ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(BaseModel): +class ReadIcebergRESTCatalogOAuth2ClientCredentialsAuthSchema(SecretDumpMixin): type: Literal["iceberg_rest_oauth2_client_credentials"] = Field(description="Auth type") rest_catalog_oauth2_client_id: str rest_catalog_oauth2_scopes: list[str] = Field(default_factory=list) diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py index a4281b4c..66802146 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/bearer.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadIcebergRESTCatalogBearerS3BasicAuthSchema(BaseModel): + +class ReadIcebergRESTCatalogBearerS3BasicAuthSchema(SecretDumpMixin): type: Literal["iceberg_rest_bearer_s3_basic"] = Field(description="Auth type") s3_access_key: str diff --git a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py index bdcb7579..6713da4e 100644 --- a/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py +++ b/syncmaster/schemas/v1/auth/iceberg_rest_s3_direct/oauth2_client_credentials.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema(BaseModel): + +class ReadIcebergRESTCatalogOAuth2ClientCredentialsS3BasicAuthSchema(SecretDumpMixin): type: Literal["iceberg_rest_oauth2_client_credentials_s3_basic"] = Field(description="Auth type") rest_catalog_oauth2_client_id: str rest_catalog_oauth2_scopes: list[str] = Field(default_factory=list) diff --git a/syncmaster/schemas/v1/auth/mixins.py b/syncmaster/schemas/v1/auth/mixins.py new file mode 100644 index 00000000..df92d1c4 --- /dev/null +++ b/syncmaster/schemas/v1/auth/mixins.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023-2024 MTS PJSC +# SPDX-License-Identifier: Apache-2.0 +from pydantic import BaseModel, SecretStr, field_serializer + + +class SecretDumpMixin(BaseModel): + + @field_serializer("*", when_used="json") + def dump_secrets(self, value): + if isinstance(value, SecretStr): + return value.get_secret_value() + return value diff --git a/syncmaster/schemas/v1/auth/s3.py b/syncmaster/schemas/v1/auth/s3.py index 2e66f779..05d07f4b 100644 --- a/syncmaster/schemas/v1/auth/s3.py +++ b/syncmaster/schemas/v1/auth/s3.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadS3AuthSchema(BaseModel): + +class ReadS3AuthSchema(SecretDumpMixin): type: Literal["s3"] = Field(description="Auth type") access_key: str diff --git a/syncmaster/schemas/v1/auth/samba.py b/syncmaster/schemas/v1/auth/samba.py index 301c5df0..79280a58 100644 --- a/syncmaster/schemas/v1/auth/samba.py +++ b/syncmaster/schemas/v1/auth/samba.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Literal -from pydantic import BaseModel, Field, SecretStr +from pydantic import Field, SecretStr +from syncmaster.schemas.v1.auth.mixins import SecretDumpMixin -class ReadSambaAuthSchema(BaseModel): + +class ReadSambaAuthSchema(SecretDumpMixin): type: Literal["samba"] = Field(description="Auth type") user: str auth_type: Literal["NTLMv1", "NTLMv2"] = "NTLMv2" diff --git a/syncmaster/server/settings/auth/keycloak.py b/syncmaster/server/settings/auth/keycloak.py index af665641..e355ff4b 100644 --- a/syncmaster/server/settings/auth/keycloak.py +++ b/syncmaster/server/settings/auth/keycloak.py @@ -25,13 +25,14 @@ class KeycloakAuthProviderSettings(BaseModel): auth: provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider - server_url: http://localhost:8080/auth - client_id: my_keycloak_client - client_secret: keycloak_client_secret - realm_name: my_realm - redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth - verify_ssl: false - scope: openid + keycloak: + server_url: http://localhost:8080/auth + client_id: my_keycloak_client + client_secret: keycloak_client_secret + realm_name: my_realm + redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth + verify_ssl: false + scope: openid """ keycloak: KeycloakSettings = Field( diff --git a/syncmaster/server/settings/auth/oauth2_gateway.py b/syncmaster/server/settings/auth/oauth2_gateway.py index 4abe4c82..d4880b17 100644 --- a/syncmaster/server/settings/auth/oauth2_gateway.py +++ b/syncmaster/server/settings/auth/oauth2_gateway.py @@ -16,13 +16,14 @@ class OAuth2GatewayProviderSettings(BaseModel): auth: provider: syncmaster.server.providers.auth.oauth2_gateway_provider.OAuth2GatewayProvider - server_url: http://localhost:8080/auth - client_id: my_keycloak_client - client_secret: keycloak_client_secret - realm_name: my_realm - redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth - verify_ssl: false - scope: openid + keycloak: + server_url: http://localhost:8080/auth + client_id: my_keycloak_client + client_secret: keycloak_client_secret + realm_name: my_realm + redirect_uri: http://localhost:8000/auth/realms/my_realm/protocol/openid-connect/auth + verify_ssl: false + scope: openid """ keycloak: KeycloakSettings = Field(