diff --git a/packages/amp/src/AdminApi.ts b/packages/amp/src/AdminApi.ts deleted file mode 100644 index a853265..0000000 --- a/packages/amp/src/AdminApi.ts +++ /dev/null @@ -1,1932 +0,0 @@ -import * as HttpApi from "@effect/platform/HttpApi" -import * as HttpApiClient from "@effect/platform/HttpApiClient" -import * as HttpApiEndpoint from "@effect/platform/HttpApiEndpoint" -import * as HttpApiError from "@effect/platform/HttpApiError" -import * as HttpApiGroup from "@effect/platform/HttpApiGroup" -import * as HttpApiSchema from "@effect/platform/HttpApiSchema" -import * as HttpClient from "@effect/platform/HttpClient" -import type * as HttpClientError from "@effect/platform/HttpClientError" -import * as HttpClientRequest from "@effect/platform/HttpClientRequest" -import type * as KeyValueStore from "@effect/platform/KeyValueStore" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import { constUndefined } from "effect/Function" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Schema from "effect/Schema" -import * as Auth from "./Auth.ts" -import * as Models from "./Models.ts" - -// ============================================================================= -// Admin API Params -// ============================================================================= - -/** - * A URL parameter for the dataset namespace. - */ -const datasetNamespaceParam = HttpApiSchema.param("namespace", Models.DatasetNamespace) - -/** - * A URL parameter for the dataset name. - */ -const datasetNameParam = HttpApiSchema.param("name", Models.DatasetName) - -/** - * A URL parameter for the dataset revision. - */ -const datasetRevisionParam = HttpApiSchema.param("revision", Models.DatasetRevision) - -/** - * A URL parameter for the unique job identifier. - */ -const jobIdParam = HttpApiSchema.param( - "jobId", - Schema.NumberFromString.annotations({ - identifier: "JobId", - description: "The unique identifier for a job." - }) -) - -// ============================================================================= -// Admin API Schemas -// ============================================================================= - -export class GetDatasetsResponse extends Schema.Class( - "Amp/AdminApi/GetDatasetsResponse" -)({ - datasets: Schema.Array(Schema.Struct({ - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - versions: Schema.Array(Models.DatasetVersion), - latestVersion: Models.DatasetVersion.pipe( - Schema.optional, - Schema.fromKey("latest_version") - ) - })) -}, { identifier: "GetDatasetsResponse" }) {} - -export class RegisterDatasetPayload extends Schema.Class( - "Amp/AdminApi/RegisterDatasetPayload" -)({ - namespace: Schema.String, - name: Schema.String, - version: Schema.optional(Schema.String), - manifest: Models.DatasetManifest -}, { identifier: "RegisterDatasetPayload" }) {} - -export class GetDatasetVersionResponse extends Schema.Class( - "Amp/AdminApi/GetDatasetVersionResponse" -)({ - kind: Models.DatasetKind, - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - revision: Models.DatasetRevision, - manifestHash: Models.DatasetHash.pipe( - Schema.propertySignature, - Schema.fromKey("manifest_hash") - ) -}, { identifier: "GetDatasetVersionResponse" }) {} - -export class GetDatasetVersionsResponse extends Schema.Class( - "Amp/AdminApi/GetDatasetVersionsResponse" -)({ - versions: Schema.Array(Models.DatasetVersion) -}, { identifier: "GetDatasetVersionsResponse" }) {} - -export class DeployDatasetPayload extends Schema.Class( - "Amp/AdminApi/DeployRequest" -)({ - endBlock: Schema.NullOr(Schema.String).pipe( - Schema.optional, - Schema.fromKey("end_block") - ), - parallelism: Schema.optional(Schema.Number) -}, { identifier: "DeployDatasetPayload" }) {} - -export class DeployDatasetResponse extends Schema.Class( - "Amp/AdminApi/DeployResponse" -)({ - jobId: Models.JobId.pipe( - Schema.propertySignature, - Schema.fromKey("job_id") - ) -}, { identifier: "DeployDatasetResponse" }) {} - -export class GetDatasetSyncProgressResponse extends Schema.Class( - "Amp/AdminApi/GetDatasetSyncProgressResponse" -)({ - namespace: Models.DatasetNamespace.pipe( - Schema.propertySignature, - Schema.fromKey("dataset_namespace") - ), - name: Models.DatasetName.pipe( - Schema.propertySignature, - Schema.fromKey("dataset_name") - ), - revision: Models.DatasetRevision, - manifestHash: Models.DatasetHash.pipe( - Schema.propertySignature, - Schema.fromKey("manifest_hash") - ), - tables: Schema.Array(Schema.Struct({ - tableName: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("table_name") - ), - currentBlock: Schema.Int.pipe( - Schema.optional, - Schema.fromKey("current_block") - ), - startBlock: Schema.Int.pipe( - Schema.optional, - Schema.fromKey("start_block") - ), - jobId: Models.JobId.pipe( - Schema.optional, - Schema.fromKey("job_id") - ), - jobStatus: Models.JobStatus.pipe( - Schema.optional, - Schema.fromKey("job_status") - ), - filesCount: Schema.Int.pipe( - Schema.propertySignature, - Schema.fromKey("files_count") - ), - totalSizeBytes: Schema.Int.pipe( - Schema.propertySignature, - Schema.fromKey("total_size_bytes") - ) - })) -}) {} - -// ============================================================================= -// Admin API Errors -// ============================================================================= - -/** - * CatalogQualifiedTable - Table reference includes a catalog qualifier. - * - * Causes: - * - SQL query contains a catalog-qualified table reference (catalog.schema.table) - * - Only dataset-qualified tables are supported (dataset.table) - * - * Applies to: - * - POST /schema - When analyzing SQL queries - * - Query operations with catalog-qualified table references - */ -export class CatalogQualifiedTable extends Schema.Class( - "Amp/AdminApi/Errors/CatalogQualifiedTable" -)({ - code: Schema.Literal("CATALOG_QUALIFIED_TABLE").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "CatalogQualifiedTable", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "CatalogQualifiedTable" -} - -/** - * DatasetNotFound - The requested dataset does not exist. - * - * Causes: - * - Dataset ID does not exist in the system - * - Dataset has been deleted - * - Dataset not yet registered - * - * Applies to: - * - GET /datasets/{id} - When dataset ID doesn't exist - * - POST /datasets/{id}/dump - When attempting to dump non-existent dataset - * - Query operations referencing non-existent datasets - */ -export class DatasetNotFound extends Schema.Class( - "Amp/AdminApi/Errors/DatasetNotFound" -)({ - code: Schema.Literal("DATASET_NOT_FOUND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "DatasetNotFound", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "DatasetNotFound" -} - -/** - * DatasetStoreError - Failure in dataset storage operations. - * - * Causes: - * - File/object store retrieval failures - * - Manifest parsing errors (TOML/JSON) - * - Unsupported dataset kind - * - Dataset name validation failures - * - Schema validation errors (missing or mismatched) - * - Provider configuration not found - * - SQL parsing failures in dataset definitions - * - * Applies to: - * - GET /datasets - Listing datasets - * - GET /datasets/{id} - Retrieving specific dataset - * - POST /datasets/{id}/dump - When loading dataset definitions - * - Query operations that access dataset metadata - */ -export class DatasetStoreError extends Schema.Class( - "Amp/AdminApi/Errors/DatasetStoreError" -)({ - code: Schema.Literal("DATASET_STORE_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "DatasetStoreError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "DatasetStoreError" -} - -/** - * DependencyAliasNotFound - Dependency alias not found in dependencies map. - * - * Causes: - * - Table reference uses an alias not provided in dependencies - * - Function reference uses an alias not provided in dependencies - * - * Applies to: - * - POST /schema - When looking up dependency aliases - */ -export class DependencyAliasNotFound extends Schema.Class( - "Amp/AdminApi/Errors/DependencyAliasNotFound" -)({ - code: Schema.Literal("DEPENDENCY_ALIAS_NOT_FOUND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "DependencyAliasNotFound", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "DependencyAliasNotFound" -} - -/** - * DependencyNotFound - Dependency not found in dataset store. - * - * Causes: - * - Referenced dependency does not exist in dataset store - * - Specified version or hash cannot be found - * - * Applies to: - * - POST /schema - When resolving dependencies - */ -export class DependencyNotFound extends Schema.Class( - "Amp/AdminApi/Errors/DependencyNotFound" -)({ - code: Schema.Literal("DEPENDENCY_NOT_FOUND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "DependencyNotFound", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "DependencyNotFound" -} - -/** - * DependencyResolution - Failed to resolve dependency. - * - * Causes: - * - Database query fails during resolution - * - * Applies to: - * - POST /schema - When resolving dependencies - */ -export class DependencyResolution extends Schema.Class( - "Amp/AdminApi/Errors/DependencyResolution" -)({ - code: Schema.Literal("DEPENDENCY_RESOLUTION").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "DependencyResolution", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "DependencyResolution" -} - -/** - * EmptyTablesAndFunctions - No tables or functions provided. - * - * Causes: - * - At least one table or function is required for schema analysis - * - * Applies to: - * - POST /schema - When both tables and functions fields are empty - */ -export class EmptyTablesAndFunctions extends Schema.Class( - "Amp/AdminApi/Errors/EmptyTablesAndFunctions" -)({ - code: Schema.Literal("EMPTY_TABLES_AND_FUNCTIONS").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "EmptyTablesAndFunctions", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "EmptyTablesAndFunctions" -} - -/** - * EthCallNotAvailable - eth_call function not available for dataset. - * - * Causes: - * - eth_call function is referenced in SQL but dataset doesn't support it - * - Dataset is not an EVM RPC dataset - * - * Applies to: - * - POST /schema - When checking eth_call availability - */ -export class EthCallNotAvailable extends Schema.Class( - "Amp/AdminApi/Errors/EthCallNotAvailable" -)({ - code: Schema.Literal("ETH_CALL_NOT_AVAILABLE").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "EthCallNotAvailable", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "EthCallNotAvailable" -} - -/** - * EthCallUdfCreationError - Failed to create ETH call UDF. - * - * Causes: - * - Invalid provider configuration for dataset - * - Provider connection issues - * - Dataset is not an EVM RPC dataset but eth_call was requested - * - * Applies to: - * - POST /schema - When creating ETH call UDFs - */ -export class EthCallUdfCreationError extends Schema.Class( - "Amp/AdminApi/Errors/EthCallUdfCreationError" -)({ - code: Schema.Literal("ETH_CALL_UDF_CREATION_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "EthCallUdfCreationError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "EthCallUdfCreationError" -} - -/** - * FunctionNotFoundInDataset - Function not found in referenced dataset. - * - * Causes: - * - SQL query references a function that doesn't exist in the dataset - * - Function name is misspelled or dataset doesn't define the function - * - * Applies to: - * - POST /schema - When resolving function references - */ -export class FunctionNotFoundInDataset extends Schema.Class( - "Amp/AdminApi/Errors/FunctionNotFoundInDataset" -)({ - code: Schema.Literal("FUNCTION_NOT_FOUND_IN_DATASET").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "FunctionNotFoundInDataset", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "FunctionNotFoundInDataset" -} - -/** - * FunctionReferenceResolution - Failed to resolve function references from SQL. - * - * Causes: - * - Unsupported DML statements encountered - * - * Applies to: - * - POST /schema - When resolving function references - */ -export class FunctionReferenceResolution extends Schema.Class( - "Amp/AdminApi/Errors/FunctionReferenceResolution" -)({ - code: Schema.Literal("FUNCTION_REFERENCE_RESOLUTION").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "FunctionReferenceResolution", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "FunctionReferenceResolution" -} - -/** - * GetDatasetError - Failed to retrieve dataset from store. - * - * Causes: - * - Dataset manifest is invalid or corrupted - * - Unsupported dataset kind - * - Storage backend errors when reading dataset - * - * Applies to: - * - POST /schema - When loading dataset definitions - */ -export class GetDatasetError extends Schema.Class( - "Amp/AdminApi/Errors/GetDatasetError" -)({ - code: Schema.Literal("GET_DATASET_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "GetDatasetError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "GetDatasetError" -} - -/** - * GetSyncProgressError - Failed to retrieve the dataset sync progress - * - * Causes: - * - Unable to resolve the dataset synchronization progress server side - */ -export class GetSyncProgressError extends Schema.Class( - "Amp/AdminApi/Errors/GetSyncProgressError" -)({ - code: Schema.Literal("GET_SYNC_PROGRESS_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "GetSyncProgressError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "GetSyncProgressError" -} - -/** - * InvalidJobId - The provided job ID is malformed or invalid. - * - * Causes: - * - Job ID contains invalid characters - * - Job ID format does not match expected pattern - * - Empty or null job ID - * - Job ID is not a valid integer - * - * Applies to: - * - GET /jobs/{id} - When ID format is invalid - * - DELETE /jobs/{id} - When ID format is invalid - * - PUT /jobs/{id}/stop - When ID format is invalid - */ -export class InvalidJobId extends Schema.Class( - "Amp/AdminApi/Errors/InvalidJobId" -)({ - code: Schema.Literal("INVALID_JOB_ID").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidJobId", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidJobId" -} - -/** - * InvalidManifest - Dataset manifest is semantically invalid. - * - * Causes: - * - Invalid dataset references in SQL views - * - Circular dependencies between datasets - * - Invalid provider references - * - Schema validation failures - * - Invalid dataset configuration - * - * Applies to: - * - POST /datasets - During manifest validation - * - Dataset initialization - * - Different from ManifestParseError (syntax vs semantics) - */ -export class InvalidManifest extends Schema.Class( - "Amp/AdminApi/Errors/InvalidManifest" -)({ - code: Schema.Literal("INVALID_MANIFEST").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidManifest", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidManifest" -} - -/** - * InvalidPathParams - Invalid request path parameters. - * - * Causes: - * - Request path parameters were malformed - */ -export class InvalidPathParams extends Schema.Class( - "Amp/AdminApi/Errors/InvalidPathParams" -)({ - code: Schema.Literal("INVALID_PATH_PARAMS").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidPathParams", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidPathParams" -} - -/** - * InvalidPayloadFormat - Invalid request payload format. - * - * Causes: - * - Request JSON is malformed or invalid - * - Required fields are missing or have wrong types - * - Dataset name or version format is invalid - * - JSON deserialization failures - * - * Applies to: - * - POST /datasets - When request body is invalid - * - POST /schema - When request payload cannot be parsed - */ -export class InvalidPayloadFormat extends Schema.Class( - "Amp/AdminApi/Errors/InvalidPayloadFormat" -)({ - code: Schema.Literal("INVALID_PAYLOAD_FORMAT").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidPayloadFormat", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidPayloadFormat" -} - -/** - * InvalidRequest - The request is malformed or contains invalid parameters. - * - * Causes: - * - Missing required request parameters - * - Invalid parameter values - * - Malformed request body - * - Invalid content type - * - Request validation failures - * - * Applies to: - * - POST /datasets - Invalid registration request - * - POST /datasets/{id}/dump - Invalid dump parameters - * - Any endpoint with request validation - */ -export class InvalidRequest extends Schema.Class( - "Amp/AdminApi/Errors/InvalidRequest" -)({ - code: Schema.Literal("INVALID_REQUEST").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidRequest", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidRequest" -} - -/** - * InvalidSelector - The provided dataset selector (name/version) is malformed or invalid. - * - * Causes: - * - Dataset name contains invalid characters or doesn't follow naming conventions - * - Dataset name is empty or malformed - * - Version syntax is invalid (e.g., malformed semver) - * - Path parameter extraction fails for dataset selection - * - * Applies to: - * - GET /datasets/{name} - When dataset name format is invalid - * - GET /datasets/{name}/versions/{version} - When name or version format is invalid - * - Any endpoint accepting dataset selector parameters - */ -export class InvalidSelector extends Schema.Class( - "Amp/AdminApi/Errors/InvalidSelector" -)({ - code: Schema.Literal("INVALID_SELECTOR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidSelector", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidSelector" -} - -/** - * InvalidTableName - Table name does not conform to SQL identifier rules. - * - * Causes: - * - Table name contains invalid characters - * - Table name doesn't follow naming conventions - * - Table name exceeds maximum length - * - * Applies to: - * - POST /schema - When analyzing SQL queries - * - Query operations with invalid table names - */ -export class InvalidTableName extends Schema.Class( - "Amp/AdminApi/Errors/InvalidTableName" -)({ - code: Schema.Literal("INVALID_TABLE_NAME").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidTableName", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidTableName" -} - -/** - * InvalidTableSql - SQL syntax error in table definition. - * - * Causes: - * - Query parsing fails - * - * Applies to: - * - POST /schema - When analyzing table SQL queries - */ -export class InvalidTableSql extends Schema.Class( - "Amp/AdminApi/Errors/InvalidTableSql" -)({ - code: Schema.Literal("INVALID_TABLE_SQL").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "InvalidTableSql", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "InvalidTableSql" -} - -/** - * JobNotFound - The requested job does not exist. - * - * Causes: - * - Job ID does not exist in the system - * - Job has been deleted - * - Job has completed and been cleaned up - * - * Applies to: - * - GET /jobs/{id} - When job ID doesn't exist - * - DELETE /jobs/{id} - When attempting to delete non-existent job - * - PUT /jobs/{id}/stop - When attempting to stop non-existent job - */ -export class JobNotFound extends Schema.Class( - "Amp/AdminApi/Errors/JobNotFound" -)({ - code: Schema.Literal("JOB_NOT_FOUND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "JobNotFound", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "JobNotFound" -} - -/** - * ManifestLinkingError - Failed to link manifest to dataset. - * - * Causes: - * - Error during manifest linking in metadata database - * - Error updating dev tag - * - Database transaction failure - * - Foreign key constraint violations - * - * Applies to: - * - POST /datasets - During manifest linking to dataset - */ -export class ManifestLinkingError extends Schema.Class( - "Amp/AdminApi/Errors/ManifestLinkingError" -)({ - code: Schema.Literal("MANIFEST_LINKING_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "ManifestLinkingError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "ManifestLinkingError" -} - -/** - * ManifestNotFound - Manifest with the provided hash not found. - * - * Causes: - * - A manifest hash was provided but the manifest doesn't exist in the system - * - The hash is valid format but no manifest is stored with that hash - * - Manifest was deleted or never registered - * - * Applies to: - * - POST /datasets - When linking to a manifest hash that doesn't exist - */ -export class ManifestNotFound extends Schema.Class( - "Amp/AdminApi/Errors/ManifestNotFound" -)({ - code: Schema.Literal("MANIFEST_NOT_FOUND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "ManifestNotFound", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "ManifestNotFound" -} - -/** - * ManifestRegistrationError - Failed to register manifest in the system. - * - * Causes: - * - Internal error during manifest registration - * - Registry service unavailable - * - Manifest storage failure - * - * Applies to: - * - POST /datasets - During manifest registration - * - POST /datasets - During manifest registration - */ -export class ManifestRegistrationError extends Schema.Class( - "Amp/AdminApi/Errors/ManifestRegistrationError" -)({ - code: Schema.Literal("MANIFEST_REGISTRATION_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "ManifestRegistrationError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "ManifestRegistrationError" -} - -/** - * ManifestValidationError - Manifest validation error. - * - * Causes: - * - SQL queries contain non-incremental operations - * - Invalid table references in SQL - * - Schema validation failures - * - Type inference errors - * - * Applies to: - * - POST /datasets - During manifest validation - */ -export class ManifestValidationError extends Schema.Class( - "Amp/AdminApi/Errors/ManifestValidationError" -)({ - code: Schema.Literal("MANIFEST_VALIDATION_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "ManifestValidationError", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "ManifestValidationError" -} - -/** - * MetadataDbError - Database operation failure in the metadata PostgreSQL database. - * - * Causes: - * - Database connection failures - * - SQL execution errors - * - Database migration issues - * - Worker notification send/receive failures - * - Data consistency errors (e.g., multiple active locations) - * - * Applies to: - * - Any operation that queries or updates metadata - * - Worker coordination operations - * - Dataset state tracking - */ -export class MetadataDbError extends Schema.Class( - "Amp/AdminApi/Errors/MetadataDbError" -)({ - code: Schema.Literal("METADATA_DB_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "MetadataDbError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "MetadataDbError" -} - -/** - * NonIncrementalQuery - SQL query contains non-incremental operations. - * - * Causes: - * - SQL contains LIMIT, ORDER BY, GROUP BY, DISTINCT, window functions - * - SQL uses outer joins (LEFT/RIGHT/FULL JOIN) - * - SQL contains recursive queries - * - * Applies to: - * - POST /schema - When validating SQL queries for incremental processing - */ -export class NonIncrementalQuery extends Schema.Class( - "Amp/AdminApi/Errors/NonIncrementalQuery" -)({ - code: Schema.Literal("NON_INCREMENTAL_QUERY").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "NonIncrementalQuery", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "NonIncrementalQuery" -} - -/** - * PhysicalTableError - Failed to access the physical table metadata - * - * Causes: - * - Failed to access the physical table metadata in the database - */ -export class PhysicalTableError extends Schema.Class( - "Amp/AdminApi/Errors/PhysicalTableError" -)({ - code: Schema.Literal("PHYSICAL_TABLE_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "PhysicalTableError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "PhysicalTableError" -} - -/** - * ResolveRevisionError - Failed to resolve the dataset revision (database error) - * - * Causes: - * - Failed to get the dataset revision from the database - */ -export class ResolveRevisionError extends Schema.Class( - "Amp/AdminApi/Errors/ResolveRevisionError" -)({ - code: Schema.Literal("RESOLVE_REVISION_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "ResolveRevisionError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "ResolveRevisionError" -} - -/** - * SchedulerError - Indicates a failure in the job scheduling system. - * - * Causes: - * - Failed to schedule a dump job - * - Worker pool unavailable - * - Internal scheduler state errors - * - * Applies to: - * - POST /datasets/{name}/dump - When scheduling dataset dumps - * - POST /datasets - When scheduling registration jobs - */ -export class SchedulerError extends Schema.Class( - "Amp/AdminApi/Errors/SchedulerError" -)({ - code: Schema.Literal("SCHEDULER_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "SchedulerError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "SchedulerError" -} - -/** - * SchemaInference - Failed to infer output schema from query. - * - * Causes: - * - Schema determination encounters errors - * - * Applies to: - * - POST /schema - When inferring output schema - */ -export class SchemaInference extends Schema.Class( - "Amp/AdminApi/Errors/SchemaInference" -)({ - code: Schema.Literal("SCHEMA_INFERENCE").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "SchemaInference", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "SchemaInference" -} - -/** - * StoreError - Dataset store operation error. - * - * Causes: - * - Failed to load dataset from store - * - Dataset store configuration errors - * - Dataset store connectivity issues - * - Object store access failures - * - * Applies to: - * - POST /datasets - During dataset store operations - */ -export class StoreError extends Schema.Class( - "Amp/AdminApi/Errors/StoreError" -)({ - code: Schema.Literal("STORE_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "StoreError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "StoreError" -} - -/** - * TableNotFoundInDataset - Table not found in dataset. - * - * Causes: - * - Table name referenced in SQL query does not exist in the dataset - * - Table name is misspelled - * - Dataset does not contain the referenced table - * - * Applies to: - * - POST /schema - When analyzing SQL queries with invalid table references - */ -export class TableNotFoundInDataset extends Schema.Class( - "Amp/AdminApi/Errors/TableNotFoundInDataset" -)({ - code: Schema.Literal("TABLE_NOT_FOUND_IN_DATASET").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "TableNotFoundInDataset", - [HttpApiSchema.AnnotationStatus]: 404 -}) { - readonly _tag = "TableNotFoundInDataset" -} - -/** - * TableReferenceResolution - Failed to extract table references from SQL. - * - * Causes: - * - Invalid table reference format encountered - * - * Applies to: - * - POST /schema - When resolving table references - */ -export class TableReferenceResolution extends Schema.Class( - "Amp/AdminApi/Errors/TableReferenceResolution" -)({ - code: Schema.Literal("TABLE_REFERENCE_RESOLUTION").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "TableReferenceResolution", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "TableReferenceResolution" -} - -/** - * UnqualifiedTable - Table reference is not qualified with a dataset. - * - * Causes: - * - SQL query contains a table reference without a schema/dataset qualifier - * - All tables must be qualified with a dataset reference (e.g., dataset.table) - * - * Applies to: - * - POST /schema - When analyzing SQL queries - * - Query operations with unqualified table references - */ -export class UnqualifiedTable extends Schema.Class( - "Amp/AdminApi/Errors/UnqualifiedTable" -)({ - code: Schema.Literal("UNQUALIFIED_TABLE").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "UnqualifiedTable", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "UnqualifiedTable" -} - -/** - * UnsupportedDatasetKind - Dataset kind is not supported. - * - * Causes: - * - Dataset kind is not one of the supported types (manifest, evm-rpc, firehose, eth-beacon) - * - Invalid or unknown dataset kind value - * - Legacy dataset kinds that are no longer supported - * - * Applies to: - * - POST /datasets - During manifest validation - */ -export class UnsupportedDatasetKind extends Schema.Class( - "Amp/AdminApi/Errors/UnsupportedDatasetKind" -)({ - code: Schema.Literal("UNSUPPORTED_DATASET_KIND").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "UnsupportedDatasetKind", - [HttpApiSchema.AnnotationStatus]: 400 -}) { - readonly _tag = "UnsupportedDatasetKind" -} - -/** - * VersionTaggingError - Failed to tag version for the dataset. - * - * Causes: - * - Error during version tagging in metadata database - * - Invalid semantic version format - * - Error updating latest tag - * - Database constraint violations - * - * Applies to: - * - POST /datasets - During version tagging - */ -export class VersionTaggingError extends Schema.Class( - "Amp/AdminApi/Errors/VersionTaggingError" -)({ - code: Schema.Literal("VERSION_TAGGING_ERROR").pipe( - Schema.propertySignature, - Schema.fromKey("error_code") - ), - message: Schema.String.pipe( - Schema.propertySignature, - Schema.fromKey("error_message") - ) -}, { - identifier: "VersionTaggingError", - [HttpApiSchema.AnnotationStatus]: 500 -}) { - readonly _tag = "VersionTaggingError" -} - -// ============================================================================= -// Admin API Endpoints -// ============================================================================= - -// ----------------------------------------------------------------------------- -// GET /datasets -// ----------------------------------------------------------------------------- - -/** - * The get datasets endpoint (GET /datasets). - */ -const getDatasets = HttpApiEndpoint.get("getDatasets")`/datasets` - .addError(DatasetStoreError) - .addError(MetadataDbError) - .addSuccess(GetDatasetsResponse) - -/** - * Error type for the `getDatasets` endpoint. - * - * - DatasetStoreError: Failed to retrieve datasets from the dataset store. - * - MetadataDbError: Database error while retrieving active locations for tables. - */ -export type GetDatasetsError = - | DatasetStoreError - | MetadataDbError - -// ----------------------------------------------------------------------------- -// POST /datasets -// ----------------------------------------------------------------------------- - -const registerDataset = HttpApiEndpoint.post("registerDataset")`/datasets` - .addError(InvalidPayloadFormat) - .addError(InvalidManifest) - .addError(ManifestLinkingError) - .addError(ManifestNotFound) - .addError(ManifestRegistrationError) - .addError(ManifestValidationError) - .addError(StoreError) - .addError(UnsupportedDatasetKind) - .addError(VersionTaggingError) - .addSuccess(Schema.Void, { status: 201 }) - .setPayload(RegisterDatasetPayload) - -/** - * Error type for the `registerDataset` endpoint. - * - * - InvalidPayloadFormat: Request JSON is malformed or invalid. - * - InvalidManifest: Manifest JSON is malformed or structurally invalid. - * - ManifestLinkingError: Failed to link manifest to dataset. - * - ManifestNotFound: Manifest hash provided but manifest doesn't exist. - * - ManifestRegistrationError: Failed to register manifest in system. - * - ManifestValidationError: Manifest validation error (e.g., non-incremental operations). - * - StoreError: Dataset store operation error. - * - UnsupportedDatasetKind: Dataset kind is not supported. - * - VersionTaggingError: Failed to tag version for the dataset. - */ -export type RegisterDatasetError = - | InvalidPayloadFormat - | InvalidManifest - | ManifestLinkingError - | ManifestNotFound - | ManifestRegistrationError - | ManifestValidationError - | StoreError - | UnsupportedDatasetKind - | VersionTaggingError - -// ----------------------------------------------------------------------------- -// GET /datasets/{namespace}/{name}/versions -// ----------------------------------------------------------------------------- - -const getDatasetVersions = HttpApiEndpoint.get( - "getDatasetVersions" -)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions` - .addError(DatasetStoreError) - .addError(InvalidRequest) - .addError(MetadataDbError) - .addSuccess(GetDatasetVersionsResponse) - -/** - * Error type for the `getDatasetVersions` endpoint. - * - * - DatasetStoreError: Failed to list version tags from dataset store. - * - InvalidRequest: Invalid namespace or name in path parameters. - * - MetadataDbError: Database error while retrieving versions. - */ -export type GetDatasetVersionsError = - | DatasetStoreError - | InvalidRequest - | MetadataDbError - -// ----------------------------------------------------------------------------- -// GET /datasets/{namespace}/{name}/versions/{revision} -// ----------------------------------------------------------------------------- - -const getDatasetVersion = HttpApiEndpoint.get( - "getDatasetVersion" -)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}` - .addError(DatasetNotFound) - .addError(DatasetStoreError) - .addError(InvalidRequest) - .addError(MetadataDbError) - .addSuccess(GetDatasetVersionResponse) - -/** - * Error type for the `getDatasetVersion` endpoint. - * - * - DatasetNotFound: The dataset or revision was not found. - * - DatasetStoreError: Failed to load dataset from store. - * - InvalidRequest: Invalid namespace, name, or revision in path parameters. - * - MetadataDbError: Database error while retrieving dataset information. - */ -export type GetDatasetVersionError = - | DatasetNotFound - | DatasetStoreError - | InvalidRequest - | MetadataDbError - -// ----------------------------------------------------------------------------- -// POST /datasets/{namespace}/{name}/versions/{revision}/deploy -// ----------------------------------------------------------------------------- - -const deployDataset = HttpApiEndpoint.post( - "deployDataset" -)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/deploy` - .addError(DatasetNotFound) - .addError(DatasetStoreError) - .addError(InvalidRequest) - .addError(MetadataDbError) - .addError(SchedulerError) - .addSuccess(DeployDatasetResponse, { status: 202 }) - .setPayload(DeployDatasetPayload) - -/** - * Error type for the `deployDataset` endpoint. - * - * - DatasetNotFound: The dataset or revision was not found. - * - DatasetStoreError: Failed to load dataset from store. - * - InvalidRequest: Invalid path parameters or request body. - * - MetadataDbError: Database error while scheduling job. - * - SchedulerError: Failed to schedule the deployment job. - */ -export type DeployDatasetError = - | DatasetNotFound - | DatasetStoreError - | InvalidRequest - | MetadataDbError - | SchedulerError - -// ----------------------------------------------------------------------------- -// GET /datasets/{namespace}/{name}/versions/{revision}/manifest -// ----------------------------------------------------------------------------- - -const getDatasetManifest = HttpApiEndpoint.get( - "getDatasetManifest" -)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/manifest` - .addError(DatasetNotFound) - .addError(DatasetStoreError) - .addError(InvalidRequest) - .addError(MetadataDbError) - .addSuccess(Models.DatasetManifest) - -/** - * Error type for the `getDatasetManifest` endpoint. - * - * - DatasetNotFound: The dataset, revision, or manifest was not found. - * - DatasetStoreError: Failed to read manifest from store. - * - InvalidRequest: Invalid namespace, name, or revision in path parameters. - * - MetadataDbError: Database error while retrieving manifest path. - */ -export type GetDatasetManifestError = - | DatasetNotFound - | DatasetStoreError - | InvalidRequest - | MetadataDbError - -// ----------------------------------------------------------------------------- -// GET /jobs/{jobId} -// ----------------------------------------------------------------------------- - -const getJobById = HttpApiEndpoint.get("getJobById")`/jobs/${jobIdParam}` - .addError(InvalidJobId) - .addError(JobNotFound) - .addError(MetadataDbError) - .addSuccess(Models.JobInfo) - -/** - * Error type for the `getJobById` endpoint. - * - * - InvalidJobId: The provided ID is not a valid job identifier. - * - JobNotFound: No job exists with the given ID. - * - MetadataDbError: Internal database error occurred. - */ -export type GetJobByIdError = - | InvalidJobId - | JobNotFound - | MetadataDbError - -// ----------------------------------------------------------------------------- -// POST /schema -// ----------------------------------------------------------------------------- - -export class GetOutputSchemaPayload extends Schema.Class("SchemaRequest")({ - tables: Schema.Record({ - key: Schema.String, - value: Schema.String - }), - dependencies: Schema.Record({ - key: Schema.String, - value: Models.DatasetReferenceFromString - }).pipe(Schema.optional), - functions: Schema.Record({ - key: Schema.String, - value: Models.FunctionDefinition - }).pipe(Schema.optional) -}) {} - -export class GetOutputSchemaResponse extends Schema.Class("SchemaResponse")({ - schemas: Schema.Record({ - key: Schema.String, - value: Models.TableSchemaWithNetworks - }) -}) {} - -/** - * The output schema endpoint (POST /schema). - */ -const getOutputSchema = HttpApiEndpoint.post("getOutputSchema")`/schema` - .addError(CatalogQualifiedTable) - .addError(DatasetNotFound) - .addError(DependencyAliasNotFound) - .addError(DependencyNotFound) - .addError(DependencyResolution) - .addError(EmptyTablesAndFunctions) - .addError(EthCallNotAvailable) - .addError(EthCallUdfCreationError) - .addError(FunctionNotFoundInDataset) - .addError(FunctionReferenceResolution) - .addError(GetDatasetError) - .addError(InvalidPayloadFormat) - .addError(InvalidTableName) - .addError(InvalidTableSql) - .addError(NonIncrementalQuery) - .addError(SchemaInference) - .addError(TableNotFoundInDataset) - .addError(TableReferenceResolution) - .addError(UnqualifiedTable) - .addSuccess(GetOutputSchemaResponse) - .setPayload(GetOutputSchemaPayload) - -/** - * Error type for the `getOutputSchema` endpoint. - * - * - CatalogQualifiedTable: Table reference includes catalog qualifier (not supported). - * - DependencyAliasNotFound: Table or function reference uses undefined alias. - * - DatasetNotFound: Referenced dataset does not exist in the store. - * - DependencyNotFound: Referenced dependency does not exist. - * - DependencyResolution: Failed to resolve dependency to hash. - * - EmptyTablesAndFunctions: No tables or functions provided (at least one required). - * - EthCallNotAvailable: eth_call function not available for dataset. - * - EthCallUdfCreationError: Failed to create ETH call UDF. - * - FunctionNotFoundInDataset: Referenced function does not exist in dataset. - * - FunctionReferenceResolution: Failed to resolve function references in SQL. - * - GetDatasetError: Failed to retrieve dataset from store. - * - InvalidPayloadFormat: Request JSON is malformed or missing required fields. - * - InvalidTableName: Table name does not conform to SQL identifier rules. - * - InvalidTableSql: SQL query has invalid syntax. - * - SchemaInference: Failed to infer schema for table. - * - TableNotFoundInDataset: Referenced table does not exist in dataset. - * - TableReferenceResolution: Failed to resolve table references in SQL. - * - UnqualifiedTable: Table reference is not qualified with dataset. - */ -export type GetOutputSchemaError = - | CatalogQualifiedTable - | DatasetNotFound - | DependencyAliasNotFound - | DependencyNotFound - | DependencyResolution - | EmptyTablesAndFunctions - | EthCallNotAvailable - | EthCallUdfCreationError - | FunctionNotFoundInDataset - | FunctionReferenceResolution - | GetDatasetError - | InvalidPayloadFormat - | InvalidTableName - | InvalidTableSql - | NonIncrementalQuery - | SchemaInference - | TableNotFoundInDataset - | TableReferenceResolution - | UnqualifiedTable - -// ----------------------------------------------------------------------------- -// GET /datasets/{namespace}/{name}/versions/{revision}/sync-progress -// ----------------------------------------------------------------------------- - -const getDatasetSyncProgress = HttpApiEndpoint.get( - "getDatasetSyncProgress" -)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/sync-progress` - .addError(DatasetNotFound) - .addError(GetDatasetError) - .addError(GetSyncProgressError) - .addError(InvalidPathParams) - .addError(ResolveRevisionError) - .addError(PhysicalTableError) - .addSuccess(GetDatasetSyncProgressResponse) - -/** - * Error type for the `getDatasetSyncProgress` endpoint. - * - * - DatasetNotFound - Dataset revision does not exist - * - GetDatasetError - Failed to retrieve dataset definition - * - GetSyncProgressError - Failed to get sync progress from metadata database - * - InvalidPathParams - Invalid path parameters (namespace, name, or revision malformed) - * - ResolveRevisionError - Failed to resolve dataset revision (database error) - * - PhysicalTableError - Failed to access physical table metadata - */ -export type GetDatasetSyncProgressError = - | DatasetNotFound - | GetDatasetError - | GetSyncProgressError - | InvalidPathParams - | ResolveRevisionError - | PhysicalTableError - -// ============================================================================= -// Admin API Groups -// ============================================================================= - -/** - * The api group for the dataset endpoints. - */ -export class DatasetGroup extends HttpApiGroup.make("dataset") - .add(registerDataset) - .add(getDatasets) - .add(getDatasetVersions) - .add(getDatasetVersion) - .add(deployDataset) - .add(getDatasetManifest) - .add(getDatasetSyncProgress) -{} - -/** - * The api group for the job endpoints. - */ -export class JobGroup extends HttpApiGroup.make("job").add(getJobById) {} - -/** - * The api group for the schema endpoints. - */ -export class SchemaGroup extends HttpApiGroup.make("schema").add(getOutputSchema) {} - -// ============================================================================= -// Admin API -// ============================================================================= - -/** - * The api definition for the admin api. - */ -export class Api extends HttpApi.make("admin") - .add(DatasetGroup) - .add(JobGroup) - .add(SchemaGroup) - .addError(HttpApiError.Forbidden) - .addError(HttpApiError.Unauthorized) -{} - -// ============================================================================= -// Admin API Service -// ============================================================================= - -/** - * Options for dumping a dataset. - */ -export interface DumpDatasetOptions { - /** - * The version of the dataset to dump. - */ - readonly version?: string | undefined - /** - * The block up to which to dump. - */ - readonly endBlock?: number | undefined -} - -/** - * Represents possible errors that can occur when performing HTTP requests. - */ -export type HttpError = - | HttpApiError.Forbidden - | HttpApiError.Unauthorized - | HttpClientError.HttpClientError - -/** - * A service which can be used to execute operations against the Amp admin API. - */ -export class AdminApi extends Context.Tag("Amp/AdminApi") Effect.Effect - - /** - * Get all datasets. - * - * @return The list of all datasets. - */ - readonly getDatasets: () => Effect.Effect - - /** - * Get all versions of a specific dataset. - * - * @param namespace The namespace of the dataset. - * @param name The name of the dataset. - * @return The list of all dataset versions. - */ - readonly getDatasetVersions: ( - namespace: Models.DatasetNamespace, - name: Models.DatasetName - ) => Effect.Effect - - /** - * Get a specific dataset version. - * - * @param namespace The namespace of the dataset. - * @param name The name of the dataset. - * @param revision The version/revision of the dataset. - * @return The dataset version information. - */ - readonly getDatasetVersion: ( - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - revision: Models.DatasetRevision - ) => Effect.Effect - - /** - * Deploy a dataset version. - * - * @param namespace The namespace of the dataset. - * @param name The name of the dataset to deploy. - * @param revision The version/revision to deploy. - * @param options The deployment options. - * @return The deployment response with job ID. - */ - readonly deployDataset: ( - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - revision: Models.DatasetRevision, - options?: { - endBlock?: string | null | undefined - parallelism?: number | undefined - } | undefined - ) => Effect.Effect - - /** - * Get the manifest for a dataset version. - * - * @param namespace The namespace of the dataset. - * @param name The name of the dataset. - * @param revision The version/revision of the dataset. - * @return The dataset manifest. - */ - readonly getDatasetManifest: ( - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - revision: Models.DatasetRevision - ) => Effect.Effect - - /** - * Retrieves sync progress information for a specific dataset revision, - * including per-table current block numbers, job status, and file statistics. - * - * @param namespace The namespace of the dataset. - * @param name The name of the dataset. - * @param revision The version/revision of the dataset. - * @return The dataset sync progress. - */ - readonly getDatasetSyncProgress: ( - namespace: Models.DatasetNamespace, - name: Models.DatasetName, - revision: Models.DatasetRevision - ) => Effect.Effect - - /** - * Get a job by ID. - * - * @param jobId The ID of the job to get. - * @return The job information. - */ - readonly getJobById: ( - jobId: number - ) => Effect.Effect - - /** - * Gets the schema of a dataset. - * - * @param request - The schema request with tables and dependencies. - * @returns An effect that resolves to the schema response. - */ - readonly getOutputSchema: ( - request: GetOutputSchemaPayload - ) => Effect.Effect -}>() {} - -export interface MakeOptions { - readonly url: string | URL -} - -const make = Effect.fnUntraced(function*(options: MakeOptions) { - type Service = typeof AdminApi.Service - - const auth = yield* Effect.serviceOption(Auth.Auth) - - const client = yield* HttpApiClient.make(Api, { - baseUrl: options.url, - transformClient: Option.match(auth, { - onNone: constUndefined, - onSome: (auth) => - HttpClient.mapRequestEffect( - Effect.fnUntraced(function*(request) { - const authInfo = yield* auth.getCachedAuthInfo - if (Option.isNone(authInfo)) return request - const token = authInfo.value.accessToken - return HttpClientRequest.bearerToken(request, token) - }) - ) - }) - }) - - // Dataset Operations - - const deployDataset: Service["deployDataset"] = Effect.fn("AdminApi.deployDataset")( - function*(namespace, name, revision, options) { - const path = { namespace, name, revision } - const payload = { - endBlock: options?.endBlock, - parallelism: options?.parallelism - } - yield* Effect.annotateCurrentSpan({ ...path, ...payload }) - return yield* client.dataset.deployDataset({ path, payload }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const getDatasetManifest: Service["getDatasetManifest"] = Effect.fn("AdminApi.getDatasetManifest")( - function*(namespace, name, revision) { - const path = { namespace, name, revision } - yield* Effect.annotateCurrentSpan(path) - return yield* client.dataset.getDatasetManifest({ path }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const getDatasets: Service["getDatasets"] = Effect.fn("AdminApi.getDatasets")( - function*() { - return yield* client.dataset.getDatasets({}) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const getDatasetVersion: Service["getDatasetVersion"] = Effect.fn("AdminApi.getDatasetVersion")( - function*(namespace, name, revision) { - const path = { namespace, name, revision } - yield* Effect.annotateCurrentSpan(path) - return yield* client.dataset.getDatasetVersion({ path }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const getDatasetVersions: Service["getDatasetVersions"] = Effect.fn("AdminApi.getDatasetVersions")( - function*(namespace, name) { - const path = { namespace, name } - yield* Effect.annotateCurrentSpan(path) - return yield* client.dataset.getDatasetVersions({ path }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const registerDataset: Service["registerDataset"] = Effect.fn("AdminApi.registerDataset")( - function*(namespace, name, manifest, version) { - const payload = { namespace, name, version, manifest } - yield* Effect.annotateCurrentSpan(payload) - return yield* client.dataset.registerDataset({ payload }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - const getDatasetSyncProgress: Service["getDatasetSyncProgress"] = Effect.fn("AdminApi.getDatasetSyncProgress")( - function*(namespace, name, revision) { - const path = { namespace, name, revision } - yield* Effect.annotateCurrentSpan(path) - return yield* client.dataset.getDatasetSyncProgress({ path }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - // Job Operations - - const getJobById: Service["getJobById"] = Effect.fn("AdminApi.getJobById")( - function*(jobId) { - const path = { jobId } - yield* Effect.annotateCurrentSpan(path) - return yield* client.job.getJobById({ path }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - // Schema Operations - - const getOutputSchema: Service["getOutputSchema"] = Effect.fn("AdminApi.getOutputSchema")( - function*(payload) { - return yield* client.schema.getOutputSchema({ payload }) - }, - Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) - ) - - return AdminApi.of({ - deployDataset, - getDatasetManifest, - getDatasets, - getDatasetVersion, - getDatasetVersions, - getDatasetSyncProgress, - registerDataset, - getJobById, - getOutputSchema - }) -}) - -/** - * Creates a layer for the Admin API service. - */ -export const layer = (options: MakeOptions): Layer.Layer< - AdminApi, - never, - HttpClient.HttpClient -> => Layer.effect(AdminApi, make(options)) - -/** - * Creates a layer for the Admin API service with authentication provided by - * default. - */ -export const layerAuth = (options: MakeOptions): Layer.Layer< - AdminApi, - never, - HttpClient.HttpClient | KeyValueStore.KeyValueStore -> => - Layer.effect(AdminApi, make(options)).pipe( - Layer.provide(Auth.layer) - ) diff --git a/packages/amp/src/admin/api.ts b/packages/amp/src/admin/api.ts new file mode 100644 index 0000000..7bb8721 --- /dev/null +++ b/packages/amp/src/admin/api.ts @@ -0,0 +1,485 @@ +/** + * This module contains the HttpApi definitions for the Amp Admin API. + * + * The Admin API provides operations for managing: + * - Datasets (registration, versioning, deployment) + * - Jobs (listing, stopping, deletion) + * - Workers (listing) + * - Providers (listing) + * - Schema analysis + * - Manifests (registration) + */ +import * as HttpApi from "@effect/platform/HttpApi" +import * as HttpApiEndpoint from "@effect/platform/HttpApiEndpoint" +import * as HttpApiError from "@effect/platform/HttpApiError" +import * as HttpApiGroup from "@effect/platform/HttpApiGroup" +import * as HttpApiSchema from "@effect/platform/HttpApiSchema" +import * as Schema from "effect/Schema" +import * as Models from "../Models.ts" +import * as Domain from "./domain.ts" +import * as Error from "./error.ts" + +// ============================================================================= +// Admin API Params +// ============================================================================= + +/** + * A URL parameter for the dataset namespace. + */ +const datasetNamespaceParam = HttpApiSchema.param("namespace", Models.DatasetNamespace) + +/** + * A URL parameter for the dataset name. + */ +const datasetNameParam = HttpApiSchema.param("name", Models.DatasetName) + +/** + * A URL parameter for the dataset revision. + */ +const datasetRevisionParam = HttpApiSchema.param("revision", Models.DatasetRevision) + +/** + * A URL parameter for the unique job identifier. + */ +const jobIdParam = HttpApiSchema.param( + "jobId", + Schema.NumberFromString.annotations({ + identifier: "JobId", + description: "The unique identifier for a job." + }) +) + +/** + * Query parameters for listing jobs. + */ +const jobsQueryParams = Schema.Struct({ + limit: Schema.NumberFromString.pipe(Schema.optional), + lastJobId: Schema.NumberFromString.pipe( + Schema.optional, + Schema.fromKey("last_job_id") + ), + status: Schema.String.pipe(Schema.optional) +}) + +// ============================================================================= +// Dataset Endpoints +// ============================================================================= + +// GET /datasets - List all datasets +const getDatasets = HttpApiEndpoint.get("getDatasets")`/datasets` + .addError(Error.DatasetStoreError) + .addError(Error.MetadataDbError) + .addError(Error.ListAllDatasetsError) + .addSuccess(Domain.GetDatasetsResponse) + +/** + * Error type for the `getDatasets` endpoint. + */ +export type GetDatasetsError = + | Error.DatasetStoreError + | Error.MetadataDbError + | Error.ListAllDatasetsError + +// POST /datasets - Register a dataset +const registerDataset = HttpApiEndpoint.post("registerDataset")`/datasets` + .addError(Error.InvalidPayloadFormatError) + .addError(Error.InvalidManifestError) + .addError(Error.ManifestLinkingError) + .addError(Error.ManifestNotFoundError) + .addError(Error.ManifestRegistrationError) + .addError(Error.ManifestValidationError) + .addError(Error.StoreError) + .addError(Error.UnsupportedDatasetKindError) + .addError(Error.VersionTaggingError) + .addSuccess(Schema.Void, { status: 201 }) + .setPayload(Domain.RegisterDatasetPayload) + +/** + * Error type for the `registerDataset` endpoint. + */ +export type RegisterDatasetError = + | Error.InvalidPayloadFormatError + | Error.InvalidManifestError + | Error.ManifestLinkingError + | Error.ManifestNotFoundError + | Error.ManifestRegistrationError + | Error.ManifestValidationError + | Error.StoreError + | Error.UnsupportedDatasetKindError + | Error.VersionTaggingError + +// GET /datasets/{namespace}/{name}/versions - List versions +const getDatasetVersions = HttpApiEndpoint.get( + "getDatasetVersions" +)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions` + .addError(Error.DatasetStoreError) + .addError(Error.InvalidRequestError) + .addError(Error.MetadataDbError) + .addError(Error.ListVersionTagsError) + .addSuccess(Domain.GetDatasetVersionsResponse) + +/** + * Error type for the `getDatasetVersions` endpoint. + */ +export type GetDatasetVersionsError = + | Error.DatasetStoreError + | Error.InvalidRequestError + | Error.MetadataDbError + | Error.ListVersionTagsError + +// GET /datasets/{namespace}/{name}/versions/{revision} - Get dataset version +const getDatasetVersion = HttpApiEndpoint.get( + "getDatasetVersion" +)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}` + .addError(Error.DatasetNotFoundError) + .addError(Error.DatasetStoreError) + .addError(Error.InvalidPathError) + .addError(Error.MetadataDbError) + .addError(Error.ResolveRevisionError) + .addError(Error.GetManifestPathError) + .addError(Error.ReadManifestError) + .addError(Error.ParseManifestError) + .addSuccess(Domain.GetDatasetVersionResponse) + +/** + * Error type for the `getDatasetVersion` endpoint. + */ +export type GetDatasetVersionError = + | Error.DatasetNotFoundError + | Error.DatasetStoreError + | Error.InvalidPathError + | Error.MetadataDbError + | Error.ResolveRevisionError + | Error.GetManifestPathError + | Error.ReadManifestError + | Error.ParseManifestError + +// POST /datasets/{namespace}/{name}/versions/{revision}/deploy - Deploy dataset +const deployDataset = HttpApiEndpoint.post( + "deployDataset" +)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/deploy` + .addError(Error.DatasetNotFoundError) + .addError(Error.DatasetStoreError) + .addError(Error.InvalidPathError) + .addError(Error.InvalidBodyError) + .addError(Error.MetadataDbError) + .addError(Error.SchedulerError) + .addError(Error.ResolveRevisionError) + .addError(Error.GetDatasetError) + .addError(Error.ListVersionTagsError) + .addError(Error.WorkerNotAvailableError) + .addSuccess(Domain.DeployDatasetResponse, { status: 202 }) + .setPayload(Domain.DeployDatasetPayload) + +/** + * Error type for the `deployDataset` endpoint. + */ +export type DeployDatasetError = + | Error.DatasetNotFoundError + | Error.DatasetStoreError + | Error.InvalidPathError + | Error.InvalidBodyError + | Error.MetadataDbError + | Error.SchedulerError + | Error.ResolveRevisionError + | Error.GetDatasetError + | Error.ListVersionTagsError + | Error.WorkerNotAvailableError + +// GET /datasets/{namespace}/{name}/versions/{revision}/manifest - Get manifest +const getDatasetManifest = HttpApiEndpoint.get( + "getDatasetManifest" +)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/manifest` + .addError(Error.DatasetNotFoundError) + .addError(Error.DatasetStoreError) + .addError(Error.InvalidPathError) + .addError(Error.MetadataDbError) + .addError(Error.GetManifestPathError) + .addError(Error.ReadManifestError) + .addError(Error.ParseManifestError) + .addSuccess(Models.DatasetManifest) + +/** + * Error type for the `getDatasetManifest` endpoint. + */ +export type GetDatasetManifestError = + | Error.DatasetNotFoundError + | Error.DatasetStoreError + | Error.InvalidPathError + | Error.MetadataDbError + | Error.GetManifestPathError + | Error.ReadManifestError + | Error.ParseManifestError + +// GET /datasets/{namespace}/{name}/versions/{revision}/sync-progress - Get sync progress +const getDatasetSyncProgress = HttpApiEndpoint.get( + "getDatasetSyncProgress" +)`/datasets/${datasetNamespaceParam}/${datasetNameParam}/versions/${datasetRevisionParam}/sync-progress` + .addError(Error.DatasetNotFoundError) + .addError(Error.GetDatasetError) + .addError(Error.GetSyncProgressError) + .addError(Error.InvalidPathParamsError) + .addError(Error.ResolveRevisionError) + .addError(Error.PhysicalTableError) + .addSuccess(Domain.GetDatasetSyncProgressResponse) + +/** + * Error type for the `getDatasetSyncProgress` endpoint. + */ +export type GetDatasetSyncProgressError = + | Error.DatasetNotFoundError + | Error.GetDatasetError + | Error.GetSyncProgressError + | Error.InvalidPathParamsError + | Error.ResolveRevisionError + | Error.PhysicalTableError + +// ============================================================================= +// Job Endpoints +// ============================================================================= + +// GET /jobs - List jobs +const getJobs = HttpApiEndpoint.get("getJobs")`/jobs` + .addError(Error.InvalidQueryParametersError) + .addError(Error.LimitTooLargeError) + .addError(Error.LimitInvalidError) + .addError(Error.ListJobsError) + .addSuccess(Domain.GetJobsResponse) + .setUrlParams(jobsQueryParams) + +/** + * Error type for the `getJobs` endpoint. + */ +export type GetJobsError = + | Error.InvalidQueryParametersError + | Error.LimitTooLargeError + | Error.LimitInvalidError + | Error.ListJobsError + +// GET /jobs/{jobId} - Get job by ID +const getJobById = HttpApiEndpoint.get("getJobById")`/jobs/${jobIdParam}` + .addError(Error.InvalidJobIdError) + .addError(Error.JobNotFoundError) + .addError(Error.GetJobError) + .addSuccess(Models.JobInfo) + +/** + * Error type for the `getJobById` endpoint. + */ +export type GetJobByIdError = + | Error.InvalidJobIdError + | Error.JobNotFoundError + | Error.GetJobError + +// PUT /jobs/{jobId}/stop - Stop job +const stopJob = HttpApiEndpoint.put("stopJob")`/jobs/${jobIdParam}/stop` + .addError(Error.InvalidJobIdError) + .addError(Error.JobNotFoundError) + .addError(Error.StopJobError) + .addError(Error.UnexpectedStateConflictError) + .addSuccess(Schema.Void, { status: 200 }) + +/** + * Error type for the `stopJob` endpoint. + */ +export type StopJobError = + | Error.InvalidJobIdError + | Error.JobNotFoundError + | Error.StopJobError + | Error.UnexpectedStateConflictError + +// DELETE /jobs/{jobId} - Delete job +const deleteJob = HttpApiEndpoint.del("deleteJob")`/jobs/${jobIdParam}` + .addError(Error.InvalidJobIdError) + .addError(Error.JobConflictError) + .addError(Error.GetJobError) + .addError(Error.DeleteJobError) + .addSuccess(Schema.Void, { status: 204 }) + +/** + * Error type for the `deleteJob` endpoint. + */ +export type DeleteJobError = + | Error.InvalidJobIdError + | Error.JobConflictError + | Error.GetJobError + | Error.DeleteJobError + +// ============================================================================= +// Worker Endpoints +// ============================================================================= + +// GET /workers - List workers +const getWorkers = HttpApiEndpoint.get("getWorkers")`/workers` + .addError(Error.SchedulerListWorkersError) + .addSuccess(Domain.GetWorkersResponse) + +/** + * Error type for the `getWorkers` endpoint. + */ +export type GetWorkersError = Error.SchedulerListWorkersError + +// ============================================================================= +// Schema Endpoints +// ============================================================================= + +// POST /schema - Analyze schema +const getOutputSchema = HttpApiEndpoint.post("getOutputSchema")`/schema` + .addError(Error.CatalogQualifiedTableError) + .addError(Error.CatalogQualifiedFunctionError) + .addError(Error.DatasetNotFoundError) + .addError(Error.DependencyAliasNotFoundError) + .addError(Error.DependencyNotFoundError) + .addError(Error.DependencyResolutionError) + .addError(Error.EmptyTablesAndFunctionsError) + .addError(Error.EthCallNotAvailableError) + .addError(Error.EthCallUdfCreationError) + .addError(Error.FunctionNotFoundInDatasetError) + .addError(Error.FunctionReferenceResolutionError) + .addError(Error.GetDatasetError) + .addError(Error.InvalidPayloadFormatError) + .addError(Error.InvalidTableNameError) + .addError(Error.InvalidTableSqlError) + .addError(Error.InvalidDependencyAliasForTableRefError) + .addError(Error.InvalidDependencyAliasForFunctionRefError) + .addError(Error.NonIncrementalQueryError) + .addError(Error.SchemaInferenceError) + .addError(Error.TableNotFoundInDatasetError) + .addError(Error.TableReferenceResolutionError) + .addError(Error.UnqualifiedTableError) + .addSuccess(Domain.GetOutputSchemaResponse) + .setPayload(Domain.GetOutputSchemaPayload) + +/** + * Error type for the `getOutputSchema` endpoint. + */ +export type GetOutputSchemaError = + | Error.CatalogQualifiedTableError + | Error.CatalogQualifiedFunctionError + | Error.DatasetNotFoundError + | Error.DependencyAliasNotFoundError + | Error.DependencyNotFoundError + | Error.DependencyResolutionError + | Error.EmptyTablesAndFunctionsError + | Error.EthCallNotAvailableError + | Error.EthCallUdfCreationError + | Error.FunctionNotFoundInDatasetError + | Error.FunctionReferenceResolutionError + | Error.GetDatasetError + | Error.InvalidPayloadFormatError + | Error.InvalidTableNameError + | Error.InvalidTableSqlError + | Error.InvalidDependencyAliasForTableRefError + | Error.InvalidDependencyAliasForFunctionRefError + | Error.NonIncrementalQueryError + | Error.SchemaInferenceError + | Error.TableNotFoundInDatasetError + | Error.TableReferenceResolutionError + | Error.UnqualifiedTableError + +// ============================================================================= +// Manifest Endpoints +// ============================================================================= + +// POST /manifests - Register manifest +const registerManifest = HttpApiEndpoint.post("registerManifest")`/manifests` + .addError(Error.InvalidPayloadFormatError) + .addError(Error.InvalidManifestError) + .addError(Error.ManifestValidationError) + .addError(Error.ManifestStorageError) + .addError(Error.ManifestRegistrationError) + .addError(Error.UnsupportedDatasetKindError) + .addSuccess(Domain.RegisterManifestResponse, { status: 201 }) + .setPayload(Schema.Any) + +/** + * Error type for the `registerManifest` endpoint. + */ +export type RegisterManifestError = + | Error.InvalidPayloadFormatError + | Error.InvalidManifestError + | Error.ManifestValidationError + | Error.ManifestStorageError + | Error.ManifestRegistrationError + | Error.UnsupportedDatasetKindError + +// ============================================================================= +// Provider Endpoints +// ============================================================================= + +// GET /providers - List providers +const getProviders = HttpApiEndpoint.get("getProviders")`/providers` + .addSuccess(Domain.GetProvidersResponse) + +// ============================================================================= +// Admin API Groups +// ============================================================================= + +/** + * The api group for the dataset endpoints. + */ +export class DatasetGroup extends HttpApiGroup.make("dataset") + .add(registerDataset) + .add(getDatasets) + .add(getDatasetVersions) + .add(getDatasetVersion) + .add(deployDataset) + .add(getDatasetManifest) + .add(getDatasetSyncProgress) +{} + +/** + * The api group for the job endpoints. + */ +export class JobGroup extends HttpApiGroup.make("job") + .add(getJobs) + .add(getJobById) + .add(stopJob) + .add(deleteJob) +{} + +/** + * The api group for the worker endpoints. + */ +export class WorkerGroup extends HttpApiGroup.make("worker") + .add(getWorkers) +{} + +/** + * The api group for the schema endpoints. + */ +export class SchemaGroup extends HttpApiGroup.make("schema") + .add(getOutputSchema) +{} + +/** + * The api group for the manifest endpoints. + */ +export class ManifestGroup extends HttpApiGroup.make("manifest") + .add(registerManifest) +{} + +/** + * The api group for the provider endpoints. + */ +export class ProviderGroup extends HttpApiGroup.make("provider") + .add(getProviders) +{} + +// ============================================================================= +// Admin API +// ============================================================================= + +/** + * The api definition for the admin api. + */ +export class Api extends HttpApi.make("admin") + .add(DatasetGroup) + .add(JobGroup) + .add(WorkerGroup) + .add(SchemaGroup) + .add(ManifestGroup) + .add(ProviderGroup) + .addError(HttpApiError.Forbidden) + .addError(HttpApiError.Unauthorized) +{} diff --git a/packages/amp/src/admin/domain.ts b/packages/amp/src/admin/domain.ts new file mode 100644 index 0000000..1d6ef22 --- /dev/null +++ b/packages/amp/src/admin/domain.ts @@ -0,0 +1,267 @@ +/** + * This module contains domain models and schemas for the Admin API requests + * and responses. + */ +import * as Schema from "effect/Schema" +import * as Models from "../Models.ts" + +// ============================================================================= +// Dataset Request/Response Schemas +// ============================================================================= + +/** + * Response schema for listing datasets. + */ +export class GetDatasetsResponse extends Schema.Class( + "Amp/AdminApi/GetDatasetsResponse" +)({ + datasets: Schema.Array(Schema.Struct({ + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + versions: Schema.Array(Models.DatasetVersion), + latestVersion: Models.DatasetVersion.pipe( + Schema.optional, + Schema.fromKey("latest_version") + ) + })) +}, { identifier: "GetDatasetsResponse" }) {} + +/** + * Request payload for registering a dataset. + */ +export class RegisterDatasetPayload extends Schema.Class( + "Amp/AdminApi/RegisterDatasetPayload" +)({ + namespace: Schema.String, + name: Schema.String, + version: Schema.optional(Schema.String), + manifest: Models.DatasetManifest +}, { identifier: "RegisterDatasetPayload" }) {} + +/** + * Response schema for getting a dataset version. + */ +export class GetDatasetVersionResponse extends Schema.Class( + "Amp/AdminApi/GetDatasetVersionResponse" +)({ + kind: Models.DatasetKind, + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + revision: Models.DatasetRevision, + manifestHash: Models.DatasetHash.pipe( + Schema.propertySignature, + Schema.fromKey("manifest_hash") + ) +}, { identifier: "GetDatasetVersionResponse" }) {} + +/** + * Response schema for listing dataset versions. + */ +export class GetDatasetVersionsResponse extends Schema.Class( + "Amp/AdminApi/GetDatasetVersionsResponse" +)({ + versions: Schema.Array(Models.DatasetVersion) +}, { identifier: "GetDatasetVersionsResponse" }) {} + +/** + * Request payload for deploying a dataset. + */ +export class DeployDatasetPayload extends Schema.Class( + "Amp/AdminApi/DeployDatasetPayload" +)({ + endBlock: Schema.NullOr(Schema.String).pipe( + Schema.optional, + Schema.fromKey("end_block") + ), + parallelism: Schema.optional(Schema.Number), + workerId: Schema.String.pipe( + Schema.optional, + Schema.fromKey("worker_id") + ) +}, { identifier: "DeployDatasetPayload" }) {} + +/** + * Response schema for deploying a dataset. + */ +export class DeployDatasetResponse extends Schema.Class( + "Amp/AdminApi/DeployDatasetResponse" +)({ + jobId: Models.JobId.pipe( + Schema.propertySignature, + Schema.fromKey("job_id") + ) +}, { identifier: "DeployDatasetResponse" }) {} + +/** + * Table sync progress information. + */ +export const TableSyncProgress = Schema.Struct({ + tableName: Schema.String.pipe( + Schema.propertySignature, + Schema.fromKey("table_name") + ), + currentBlock: Schema.Int.pipe( + Schema.optional, + Schema.fromKey("current_block") + ), + startBlock: Schema.Int.pipe( + Schema.optional, + Schema.fromKey("start_block") + ), + jobId: Models.JobId.pipe( + Schema.optional, + Schema.fromKey("job_id") + ), + jobStatus: Models.JobStatus.pipe( + Schema.optional, + Schema.fromKey("job_status") + ), + filesCount: Schema.Int.pipe( + Schema.propertySignature, + Schema.fromKey("files_count") + ), + totalSizeBytes: Schema.Int.pipe( + Schema.propertySignature, + Schema.fromKey("total_size_bytes") + ) +}).annotations({ identifier: "TableSyncProgress" }) +export type TableSyncProgress = typeof TableSyncProgress.Type + +/** + * Response schema for getting dataset sync progress. + */ +export class GetDatasetSyncProgressResponse extends Schema.Class( + "Amp/AdminApi/GetDatasetSyncProgressResponse" +)({ + namespace: Models.DatasetNamespace.pipe( + Schema.propertySignature, + Schema.fromKey("dataset_namespace") + ), + name: Models.DatasetName.pipe( + Schema.propertySignature, + Schema.fromKey("dataset_name") + ), + revision: Models.DatasetRevision, + manifestHash: Models.DatasetHash.pipe( + Schema.propertySignature, + Schema.fromKey("manifest_hash") + ), + tables: Schema.Array(TableSyncProgress) +}) {} + +// ============================================================================= +// Job Request/Response Schemas +// ============================================================================= + +/** + * Response schema for listing jobs. + */ +export class GetJobsResponse extends Schema.Class( + "Amp/AdminApi/GetJobsResponse" +)({ + jobs: Schema.Array(Models.JobInfo), + nextCursor: Models.JobId.pipe( + Schema.optional, + Schema.fromKey("next_cursor") + ) +}, { identifier: "GetJobsResponse" }) {} + +// ============================================================================= +// Schema Request/Response Schemas +// ============================================================================= + +/** + * Request payload for schema analysis. + */ +export class GetOutputSchemaPayload extends Schema.Class( + "Amp/AdminApi/GetOutputSchemaPayload" +)({ + tables: Schema.Record({ + key: Schema.String, + value: Schema.String + }), + dependencies: Schema.Record({ + key: Schema.String, + value: Models.DatasetReferenceFromString + }).pipe(Schema.optional), + functions: Schema.Record({ + key: Schema.String, + value: Models.FunctionDefinition + }).pipe(Schema.optional) +}, { identifier: "GetOutputSchemaPayload" }) {} + +/** + * Response schema for schema analysis. + */ +export class GetOutputSchemaResponse extends Schema.Class( + "Amp/AdminApi/GetOutputSchemaResponse" +)({ + schemas: Schema.Record({ + key: Schema.String, + value: Models.TableSchemaWithNetworks + }) +}, { identifier: "GetOutputSchemaResponse" }) {} + +// ============================================================================= +// Worker Request/Response Schemas +// ============================================================================= + +/** + * Worker information returned by the API. + */ +export const WorkerInfo = Schema.Struct({ + nodeId: Schema.String.pipe( + Schema.propertySignature, + Schema.fromKey("node_id") + ), + heartbeatAt: Schema.String.pipe( + Schema.propertySignature, + Schema.fromKey("heartbeat_at") + ) +}).annotations({ identifier: "WorkerInfo" }) +export type WorkerInfo = typeof WorkerInfo.Type + +/** + * Response schema for listing workers. + */ +export class GetWorkersResponse extends Schema.Class( + "Amp/AdminApi/GetWorkersResponse" +)({ + workers: Schema.Array(WorkerInfo) +}, { identifier: "GetWorkersResponse" }) {} + +// ============================================================================= +// Manifest Request/Response Schemas +// ============================================================================= + +/** + * Response schema for registering a manifest. + */ +export class RegisterManifestResponse extends Schema.Class( + "Amp/AdminApi/RegisterManifestResponse" +)({ + hash: Models.DatasetHash +}, { identifier: "RegisterManifestResponse" }) {} + +// ============================================================================= +// Provider Request/Response Schemas +// ============================================================================= + +/** + * Provider information returned by the API. + */ +export const ProviderInfo = Schema.Struct({ + name: Schema.String, + network: Models.Network, + config: Schema.Any +}).annotations({ identifier: "ProviderInfo" }) +export type ProviderInfo = typeof ProviderInfo.Type + +/** + * Response schema for listing providers. + */ +export class GetProvidersResponse extends Schema.Class( + "Amp/AdminApi/GetProvidersResponse" +)({ + providers: Schema.Array(ProviderInfo) +}, { identifier: "GetProvidersResponse" }) {} diff --git a/packages/amp/src/admin/error.ts b/packages/amp/src/admin/error.ts new file mode 100644 index 0000000..96f4aef --- /dev/null +++ b/packages/amp/src/admin/error.ts @@ -0,0 +1,760 @@ +/** + * This module contains error definitions which represent the standard error + * responses returned by the Amp Admin API. + * + * Errors provide structured error details including a machine-readable error + * code and a human-readable message. + * + * ## Error Code Conventions + * - Error codes use SCREAMING_SNAKE_CASE (e.g., `DATASET_NOT_FOUND`) + * - Codes are stable and can be relied upon programmatically + * - Messages may change and should only be used for display/logging + * + * ## Example JSON Response + * ```json + * { + * "error_code": "DATASET_NOT_FOUND", + * "error_message": "dataset 'eth_mainnet' version '1.0.0' not found" + * } + * ``` + */ +import * as HttpApiSchema from "@effect/platform/HttpApiSchema" +import * as Schema from "effect/Schema" + +/** + * Machine-readable error code in SCREAMING_SNAKE_CASE format + * + * Error codes are stable across API versions and should be used + * for programmatic error handling. Examples: `INVALID_SELECTOR`, + * `DATASET_NOT_FOUND`, `METADATA_DB_ERROR` + */ +const ErrorCode = ( + code: Code +): Schema.PropertySignature<":", Code, "error_code", ":", Code> => + Schema.Literal(code).pipe( + Schema.propertySignature, + Schema.fromKey("error_code") + ) + +const BaseErrorFields = { + /** + * Human-readable error message + * + * Messages provide detailed context about the error but may change + * over time. Use `error_code` for programmatic decisions. + */ + message: Schema.String.pipe( + Schema.propertySignature, + Schema.fromKey("error_message") + ) +} + +// ============================================================================= +// Dataset Errors +// ============================================================================= + +/** + * CatalogQualifiedTable - Table reference includes a catalog qualifier. + * + * Causes: + * - SQL query contains a catalog-qualified table reference (catalog.schema.table) + * - Only dataset-qualified tables are supported (dataset.table) + */ +export class CatalogQualifiedTableError extends Schema.Class( + "Amp/AdminApi/CatalogQualifiedTableError" +)({ + ...BaseErrorFields, + code: ErrorCode("CATALOG_QUALIFIED_TABLE") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * CatalogQualifiedFunction - Function reference includes a catalog qualifier. + * + * Causes: + * - SQL query contains a catalog-qualified function reference (catalog.schema.function) + * - Only dataset-qualified functions are supported (dataset.function) + */ +export class CatalogQualifiedFunctionError extends Schema.Class( + "Amp/AdminApi/CatalogQualifiedFunctionError" +)({ + ...BaseErrorFields, + code: ErrorCode("CATALOG_QUALIFIED_FUNCTION") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * DatasetNotFound - The requested dataset does not exist. + * + * Causes: + * - Dataset ID does not exist in the system + * - Dataset has been deleted + * - Dataset not yet registered + */ +export class DatasetNotFoundError extends Schema.Class( + "Amp/AdminApi/DatasetNotFoundError" +)({ + ...BaseErrorFields, + code: ErrorCode("DATASET_NOT_FOUND") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * DatasetStoreError - Failure in dataset storage operations. + * + * Causes: + * - File/object store retrieval failures + * - Manifest parsing errors (TOML/JSON) + * - Unsupported dataset kind + * - Dataset name validation failures + */ +export class DatasetStoreError extends Schema.Class( + "Amp/AdminApi/DatasetStoreError" +)({ + ...BaseErrorFields, + code: ErrorCode("DATASET_STORE_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * DependencyAliasNotFound - Dependency alias not found in dependencies map. + * + * Causes: + * - Table reference uses an alias not provided in dependencies + * - Function reference uses an alias not provided in dependencies + */ +export class DependencyAliasNotFoundError extends Schema.Class( + "Amp/AdminApi/DependencyAliasNotFoundError" +)({ + ...BaseErrorFields, + code: ErrorCode("DEPENDENCY_ALIAS_NOT_FOUND") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * DependencyNotFound - Dependency not found in dataset store. + * + * Causes: + * - Referenced dependency does not exist in dataset store + * - Specified version or hash cannot be found + */ +export class DependencyNotFoundError extends Schema.Class( + "Amp/AdminApi/DependencyNotFoundError" +)({ + ...BaseErrorFields, + code: ErrorCode("DEPENDENCY_NOT_FOUND") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * DependencyResolution - Failed to resolve dependency. + * + * Causes: + * - Database query fails during resolution + */ +export class DependencyResolutionError extends Schema.Class( + "Amp/AdminApi/DependencyResolutionError" +)({ + ...BaseErrorFields, + code: ErrorCode("DEPENDENCY_RESOLUTION") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * EmptyTablesAndFunctions - No tables or functions provided. + * + * Causes: + * - At least one table or function is required for schema analysis + */ +export class EmptyTablesAndFunctionsError extends Schema.Class( + "Amp/AdminApi/EmptyTablesAndFunctionsError" +)({ + ...BaseErrorFields, + code: ErrorCode("EMPTY_TABLES_AND_FUNCTIONS") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * EthCallNotAvailable - eth_call function not available for dataset. + * + * Causes: + * - eth_call function is referenced in SQL but dataset doesn't support it + * - Dataset is not an EVM RPC dataset + */ +export class EthCallNotAvailableError extends Schema.Class( + "Amp/AdminApi/EthCallNotAvailableError" +)({ + ...BaseErrorFields, + code: ErrorCode("ETH_CALL_NOT_AVAILABLE") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * EthCallUdfCreationError - Failed to create ETH call UDF. + * + * Causes: + * - Invalid provider configuration for dataset + * - Provider connection issues + */ +export class EthCallUdfCreationError extends Schema.Class( + "Amp/AdminApi/EthCallUdfCreationError" +)({ + ...BaseErrorFields, + code: ErrorCode("ETH_CALL_UDF_CREATION_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * FunctionNotFoundInDataset - Function not found in referenced dataset. + * + * Causes: + * - SQL query references a function that doesn't exist in the dataset + * - Function name is misspelled + */ +export class FunctionNotFoundInDatasetError extends Schema.Class( + "Amp/AdminApi/FunctionNotFoundInDatasetError" +)({ + ...BaseErrorFields, + code: ErrorCode("FUNCTION_NOT_FOUND_IN_DATASET") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * FunctionReferenceResolution - Failed to resolve function references from SQL. + * + * Causes: + * - Unsupported DML statements encountered + */ +export class FunctionReferenceResolutionError extends Schema.Class( + "Amp/AdminApi/FunctionReferenceResolutionError" +)({ + ...BaseErrorFields, + code: ErrorCode("FUNCTION_REFERENCE_RESOLUTION") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * GetDatasetError - Failed to retrieve dataset from store. + * + * Causes: + * - Dataset manifest is invalid or corrupted + * - Unsupported dataset kind + * - Storage backend errors when reading dataset + */ +export class GetDatasetError extends Schema.Class( + "Amp/AdminApi/GetDatasetError" +)({ + ...BaseErrorFields, + code: ErrorCode("GET_DATASET_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * GetManifestPathError - Failed to query manifest path from metadata database. + */ +export class GetManifestPathError extends Schema.Class( + "Amp/AdminApi/GetManifestPathError" +)({ + ...BaseErrorFields, + code: ErrorCode("GET_MANIFEST_PATH_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * GetSyncProgressError - Failed to retrieve the dataset sync progress + * + * Causes: + * - Unable to resolve the dataset synchronization progress server side + */ +export class GetSyncProgressError extends Schema.Class( + "Amp/AdminApi/GetSyncProgressError" +)({ + ...BaseErrorFields, + code: ErrorCode("GET_SYNC_PROGRESS_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +// ============================================================================= +// Job Errors +// ============================================================================= + +/** + * InvalidJobId - The provided job ID is malformed or invalid. + * + * Causes: + * - Job ID contains invalid characters + * - Job ID format does not match expected pattern + */ +export class InvalidJobIdError extends Schema.Class( + "Amp/AdminApi/InvalidJobIdError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_JOB_ID") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * JobNotFound - The requested job does not exist. + * + * Causes: + * - Job ID does not exist in the system + * - Job has been deleted + */ +export class JobNotFoundError extends Schema.Class( + "Amp/AdminApi/JobNotFoundError" +)({ + ...BaseErrorFields, + code: ErrorCode("JOB_NOT_FOUND") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * JobConflict - Job exists but cannot be deleted (not in terminal state). + */ +export class JobConflictError extends Schema.Class( + "Amp/AdminApi/JobConflictError" +)({ + ...BaseErrorFields, + code: ErrorCode("JOB_CONFLICT") +}, HttpApiSchema.annotations({ status: 409 })) {} + +/** + * GetJobError - Failed to retrieve job from scheduler. + */ +export class GetJobError extends Schema.Class( + "Amp/AdminApi/GetJobError" +)({ + ...BaseErrorFields, + code: ErrorCode("GET_JOB_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * DeleteJobError - Failed to delete job from scheduler. + */ +export class DeleteJobError extends Schema.Class( + "Amp/AdminApi/DeleteJobError" +)({ + ...BaseErrorFields, + code: ErrorCode("DELETE_JOB_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * StopJobError - Database error during stop operation. + */ +export class StopJobError extends Schema.Class( + "Amp/AdminApi/StopJobError" +)({ + ...BaseErrorFields, + code: ErrorCode("STOP_JOB_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ListJobsError - Failed to list jobs from scheduler. + */ +export class ListJobsError extends Schema.Class( + "Amp/AdminApi/ListJobsError" +)({ + ...BaseErrorFields, + code: ErrorCode("LIST_JOBS_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * UnexpectedStateConflict - Internal state machine error. + */ +export class UnexpectedStateConflictError extends Schema.Class( + "Amp/AdminApi/UnexpectedStateConflictError" +)({ + ...BaseErrorFields, + code: ErrorCode("UNEXPECTED_STATE_CONFLICT") +}, HttpApiSchema.annotations({ status: 500 })) {} + +// ============================================================================= +// Manifest Errors +// ============================================================================= + +/** + * InvalidManifest - Dataset manifest is semantically invalid. + * + * Causes: + * - Invalid dataset references in SQL views + * - Circular dependencies between datasets + * - Schema validation failures + */ +export class InvalidManifestError extends Schema.Class( + "Amp/AdminApi/InvalidManifestError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_MANIFEST") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * ManifestLinkingError - Failed to link manifest to dataset. + */ +export class ManifestLinkingError extends Schema.Class( + "Amp/AdminApi/ManifestLinkingError" +)({ + ...BaseErrorFields, + code: ErrorCode("MANIFEST_LINKING_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ManifestNotFound - Manifest with the provided hash not found. + */ +export class ManifestNotFoundError extends Schema.Class( + "Amp/AdminApi/ManifestNotFoundError" +)({ + ...BaseErrorFields, + code: ErrorCode("MANIFEST_NOT_FOUND") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * ManifestRegistrationError - Failed to register manifest in the system. + */ +export class ManifestRegistrationError extends Schema.Class( + "Amp/AdminApi/ManifestRegistrationError" +)({ + ...BaseErrorFields, + code: ErrorCode("MANIFEST_REGISTRATION_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ManifestValidationError - Manifest validation error. + * + * Causes: + * - SQL queries contain non-incremental operations + * - Invalid table references in SQL + * - Type inference errors + */ +export class ManifestValidationError extends Schema.Class( + "Amp/AdminApi/ManifestValidationError" +)({ + ...BaseErrorFields, + code: ErrorCode("MANIFEST_VALIDATION_ERROR") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * ManifestStorageError - Failed to write manifest to object store. + */ +export class ManifestStorageError extends Schema.Class( + "Amp/AdminApi/ManifestStorageError" +)({ + ...BaseErrorFields, + code: ErrorCode("MANIFEST_STORAGE_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ParseManifestError - Failed to parse manifest JSON. + */ +export class ParseManifestError extends Schema.Class( + "Amp/AdminApi/ParseManifestError" +)({ + ...BaseErrorFields, + code: ErrorCode("PARSE_MANIFEST_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ReadManifestError - Failed to read manifest from object store. + */ +export class ReadManifestError extends Schema.Class( + "Amp/AdminApi/ReadManifestError" +)({ + ...BaseErrorFields, + code: ErrorCode("READ_MANIFEST_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +// ============================================================================= +// Request Validation Errors +// ============================================================================= + +/** + * InvalidPath - Invalid path parameters. + */ +export class InvalidPathError extends Schema.Class( + "Amp/AdminApi/InvalidPathError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_PATH") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidBody - Invalid request body. + */ +export class InvalidBodyError extends Schema.Class( + "Amp/AdminApi/InvalidBodyError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_BODY") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidPathParams - Invalid request path parameters. + */ +export class InvalidPathParamsError extends Schema.Class( + "Amp/AdminApi/InvalidPathParamsError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_PATH_PARAMS") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidPayloadFormat - Invalid request payload format. + */ +export class InvalidPayloadFormatError extends Schema.Class( + "Amp/AdminApi/InvalidPayloadFormatError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_PAYLOAD_FORMAT") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidQueryParameters - Invalid query parameters. + */ +export class InvalidQueryParametersError extends Schema.Class( + "Amp/AdminApi/InvalidQueryParametersError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_QUERY_PARAMETERS") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidRequest - The request is malformed or contains invalid parameters. + */ +export class InvalidRequestError extends Schema.Class( + "Amp/AdminApi/InvalidRequestError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_REQUEST") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidSelector - The provided dataset selector is malformed or invalid. + */ +export class InvalidSelectorError extends Schema.Class( + "Amp/AdminApi/InvalidSelectorError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_SELECTOR") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidTableName - Table name does not conform to SQL identifier rules. + */ +export class InvalidTableNameError extends Schema.Class( + "Amp/AdminApi/InvalidTableNameError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_TABLE_NAME") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidTableSql - SQL syntax error in table definition. + */ +export class InvalidTableSqlError extends Schema.Class( + "Amp/AdminApi/InvalidTableSqlError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_TABLE_SQL") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidDependencyAliasForTableRef - Invalid dependency alias in table reference. + */ +export class InvalidDependencyAliasForTableRefError extends Schema.Class( + "Amp/AdminApi/InvalidDependencyAliasForTableRefError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_DEPENDENCY_ALIAS_FOR_TABLE_REF") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * InvalidDependencyAliasForFunctionRef - Invalid dependency alias in function reference. + */ +export class InvalidDependencyAliasForFunctionRefError extends Schema.Class( + "Amp/AdminApi/InvalidDependencyAliasForFunctionRefError" +)({ + ...BaseErrorFields, + code: ErrorCode("INVALID_DEPENDENCY_ALIAS_FOR_FUNCTION_REF") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * LimitTooLarge - The requested limit exceeds the maximum allowed value. + */ +export class LimitTooLargeError extends Schema.Class( + "Amp/AdminApi/LimitTooLargeError" +)({ + ...BaseErrorFields, + code: ErrorCode("LIMIT_TOO_LARGE") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * LimitInvalid - The requested limit is invalid (zero). + */ +export class LimitInvalidError extends Schema.Class( + "Amp/AdminApi/LimitInvalidError" +)({ + ...BaseErrorFields, + code: ErrorCode("LIMIT_INVALID") +}, HttpApiSchema.annotations({ status: 400 })) {} + +// ============================================================================= +// Database Errors +// ============================================================================= + +/** + * MetadataDbError - Database operation failure in the metadata PostgreSQL database. + */ +export class MetadataDbError extends Schema.Class( + "Amp/AdminApi/MetadataDbError" +)({ + ...BaseErrorFields, + code: ErrorCode("METADATA_DB_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * PhysicalTableError - Failed to access the physical table metadata. + */ +export class PhysicalTableError extends Schema.Class( + "Amp/AdminApi/PhysicalTableError" +)({ + ...BaseErrorFields, + code: ErrorCode("PHYSICAL_TABLE_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ResolveRevisionError - Failed to resolve the dataset revision. + */ +export class ResolveRevisionError extends Schema.Class( + "Amp/AdminApi/ResolveRevisionError" +)({ + ...BaseErrorFields, + code: ErrorCode("RESOLVE_REVISION_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +// ============================================================================= +// Query/Schema Errors +// ============================================================================= + +/** + * NonIncrementalQuery - SQL query contains non-incremental operations. + * + * Causes: + * - SQL contains LIMIT, ORDER BY, GROUP BY, DISTINCT, window functions + * - SQL uses outer joins + */ +export class NonIncrementalQueryError extends Schema.Class( + "Amp/AdminApi/NonIncrementalQueryError" +)({ + ...BaseErrorFields, + code: ErrorCode("NON_INCREMENTAL_QUERY") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * SchemaInference - Failed to infer output schema from query. + */ +export class SchemaInferenceError extends Schema.Class( + "Amp/AdminApi/SchemaInferenceError" +)({ + ...BaseErrorFields, + code: ErrorCode("SCHEMA_INFERENCE") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * TableNotFoundInDataset - Table not found in dataset. + */ +export class TableNotFoundInDatasetError extends Schema.Class( + "Amp/AdminApi/TableNotFoundInDatasetError" +)({ + ...BaseErrorFields, + code: ErrorCode("TABLE_NOT_FOUND_IN_DATASET") +}, HttpApiSchema.annotations({ status: 404 })) {} + +/** + * TableReferenceResolution - Failed to extract table references from SQL. + */ +export class TableReferenceResolutionError extends Schema.Class( + "Amp/AdminApi/TableReferenceResolutionError" +)({ + ...BaseErrorFields, + code: ErrorCode("TABLE_REFERENCE_RESOLUTION") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * UnqualifiedTable - Table reference is not qualified with a dataset. + */ +export class UnqualifiedTableError extends Schema.Class( + "Amp/AdminApi/UnqualifiedTableError" +)({ + ...BaseErrorFields, + code: ErrorCode("UNQUALIFIED_TABLE") +}, HttpApiSchema.annotations({ status: 400 })) {} + +// ============================================================================= +// Scheduler/Worker Errors +// ============================================================================= + +/** + * SchedulerError - Indicates a failure in the job scheduling system. + */ +export class SchedulerError extends Schema.Class( + "Amp/AdminApi/SchedulerError" +)({ + ...BaseErrorFields, + code: ErrorCode("SCHEDULER_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * WorkerNotAvailable - Specified worker not found or inactive. + */ +export class WorkerNotAvailableError extends Schema.Class( + "Amp/AdminApi/WorkerNotAvailableError" +)({ + ...BaseErrorFields, + code: ErrorCode("WORKER_NOT_AVAILABLE") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * SchedulerListWorkersError - Failed to list workers from the scheduler. + */ +export class SchedulerListWorkersError extends Schema.Class( + "Amp/AdminApi/SchedulerListWorkersError" +)({ + ...BaseErrorFields, + code: ErrorCode("SCHEDULER_LIST_WORKERS_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +// ============================================================================= +// Store Errors +// ============================================================================= + +/** + * StoreError - Dataset store operation error. + */ +export class StoreError extends Schema.Class( + "Amp/AdminApi/StoreError" +)({ + ...BaseErrorFields, + code: ErrorCode("STORE_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * UnsupportedDatasetKind - Dataset kind is not supported. + */ +export class UnsupportedDatasetKindError extends Schema.Class( + "Amp/AdminApi/UnsupportedDatasetKindError" +)({ + ...BaseErrorFields, + code: ErrorCode("UNSUPPORTED_DATASET_KIND") +}, HttpApiSchema.annotations({ status: 400 })) {} + +/** + * VersionTaggingError - Failed to tag version for the dataset. + */ +export class VersionTaggingError extends Schema.Class( + "Amp/AdminApi/VersionTaggingError" +)({ + ...BaseErrorFields, + code: ErrorCode("VERSION_TAGGING_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ListAllDatasetsError - Failed to list all datasets from dataset store. + */ +export class ListAllDatasetsError extends Schema.Class( + "Amp/AdminApi/ListAllDatasetsError" +)({ + ...BaseErrorFields, + code: ErrorCode("LIST_ALL_DATASETS_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} + +/** + * ListVersionTagsError - Failed to list version tags from dataset store. + */ +export class ListVersionTagsError extends Schema.Class( + "Amp/AdminApi/ListVersionTagsError" +)({ + ...BaseErrorFields, + code: ErrorCode("LIST_VERSION_TAGS_ERROR") +}, HttpApiSchema.annotations({ status: 500 })) {} diff --git a/packages/amp/src/admin/service.ts b/packages/amp/src/admin/service.ts new file mode 100644 index 0000000..8846efb --- /dev/null +++ b/packages/amp/src/admin/service.ts @@ -0,0 +1,460 @@ +/** + * This module provides the AdminApi service for interacting with the Amp Admin API. + * + * The Admin API allows managing: + * - Datasets (registration, versioning, deployment, sync progress) + * - Jobs (listing, stopping, deletion) + * - Workers (listing) + * - Providers (listing) + * - Schema analysis + * - Manifests (registration) + */ +import * as HttpApiClient from "@effect/platform/HttpApiClient" +import type * as HttpApiError from "@effect/platform/HttpApiError" +import * as HttpClient from "@effect/platform/HttpClient" +import type * as HttpClientError from "@effect/platform/HttpClientError" +import * as HttpClientRequest from "@effect/platform/HttpClientRequest" +import type * as KeyValueStore from "@effect/platform/KeyValueStore" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import { constUndefined } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as Auth from "../Auth.ts" +import type * as Models from "../Models.ts" +import * as AdminApiDefinition from "./api.ts" +import type * as AdminApiDomain from "./domain.ts" + +// ============================================================================= +// Admin API Service Types +// ============================================================================= + +/** + * Represents possible errors that can occur when performing HTTP requests. + */ +export type HttpError = + | HttpApiError.Forbidden + | HttpApiError.Unauthorized + | HttpClientError.HttpClientError + +// ============================================================================= +// Admin API Service +// ============================================================================= + +/** + * A service which can be used to execute operations against the Amp admin API. + */ +export class AdminApi extends Context.Tag("Amp/AdminApi") Effect.Effect + + /** + * Get all datasets. + * + * @return The list of all datasets. + */ + readonly getDatasets: () => Effect.Effect< + AdminApiDomain.GetDatasetsResponse, + HttpError | AdminApiDefinition.GetDatasetsError + > + + /** + * Get all versions of a specific dataset. + * + * @param namespace The namespace of the dataset. + * @param name The name of the dataset. + * @return The list of all dataset versions. + */ + readonly getDatasetVersions: ( + namespace: Models.DatasetNamespace, + name: Models.DatasetName + ) => Effect.Effect< + AdminApiDomain.GetDatasetVersionsResponse, + HttpError | AdminApiDefinition.GetDatasetVersionsError + > + + /** + * Get a specific dataset version. + * + * @param namespace The namespace of the dataset. + * @param name The name of the dataset. + * @param revision The version/revision of the dataset. + * @return The dataset version information. + */ + readonly getDatasetVersion: ( + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + revision: Models.DatasetRevision + ) => Effect.Effect< + AdminApiDomain.GetDatasetVersionResponse, + HttpError | AdminApiDefinition.GetDatasetVersionError + > + + /** + * Deploy a dataset version. + * + * @param namespace The namespace of the dataset. + * @param name The name of the dataset to deploy. + * @param revision The version/revision to deploy. + * @param options The deployment options. + * @return The deployment response with job ID. + */ + readonly deployDataset: ( + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + revision: Models.DatasetRevision, + options?: { + endBlock?: string | null | undefined + parallelism?: number | undefined + workerId?: string | undefined + } | undefined + ) => Effect.Effect< + AdminApiDomain.DeployDatasetResponse, + HttpError | AdminApiDefinition.DeployDatasetError + > + + /** + * Get the manifest for a dataset version. + * + * @param namespace The namespace of the dataset. + * @param name The name of the dataset. + * @param revision The version/revision of the dataset. + * @return The dataset manifest. + */ + readonly getDatasetManifest: ( + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + revision: Models.DatasetRevision + ) => Effect.Effect + + /** + * Retrieves sync progress information for a specific dataset revision, + * including per-table current block numbers, job status, and file statistics. + * + * @param namespace The namespace of the dataset. + * @param name The name of the dataset. + * @param revision The version/revision of the dataset. + * @return The dataset sync progress. + */ + readonly getDatasetSyncProgress: ( + namespace: Models.DatasetNamespace, + name: Models.DatasetName, + revision: Models.DatasetRevision + ) => Effect.Effect< + AdminApiDomain.GetDatasetSyncProgressResponse, + HttpError | AdminApiDefinition.GetDatasetSyncProgressError + > + + /** + * Get all jobs with optional pagination and filtering. + * + * @param options Pagination and filtering options. + * @return The list of jobs with pagination cursor. + */ + readonly getJobs: (options?: { + limit?: number | undefined + lastJobId?: number | undefined + status?: string | undefined + }) => Effect.Effect< + AdminApiDomain.GetJobsResponse, + HttpError | AdminApiDefinition.GetJobsError + > + + /** + * Get a job by ID. + * + * @param jobId The ID of the job to get. + * @return The job information. + */ + readonly getJobById: ( + jobId: number + ) => Effect.Effect + + /** + * Stop a job by ID. + * + * @param jobId The ID of the job to stop. + * @return Void on success. + */ + readonly stopJob: ( + jobId: number + ) => Effect.Effect + + /** + * Delete a job by ID. + * + * @param jobId The ID of the job to delete. + * @return Void on success. + */ + readonly deleteJob: ( + jobId: number + ) => Effect.Effect + + /** + * Get all workers. + * + * @return The list of workers. + */ + readonly getWorkers: () => Effect.Effect< + AdminApiDomain.GetWorkersResponse, + HttpError | AdminApiDefinition.GetWorkersError + > + + /** + * Get all providers. + * + * @return The list of providers. + */ + readonly getProviders: () => Effect.Effect + + /** + * Register a manifest. + * + * @param manifest The manifest to register. + * @return The registered manifest response with hash. + */ + readonly registerManifest: ( + manifest: unknown + ) => Effect.Effect< + AdminApiDomain.RegisterManifestResponse, + HttpError | AdminApiDefinition.RegisterManifestError + > + + /** + * Gets the schema of a dataset. + * + * @param request - The schema request with tables and dependencies. + * @returns An effect that resolves to the schema response. + */ + readonly getOutputSchema: ( + request: AdminApiDomain.GetOutputSchemaPayload + ) => Effect.Effect< + AdminApiDomain.GetOutputSchemaResponse, + HttpError | AdminApiDefinition.GetOutputSchemaError + > +}>() {} + +export interface MakeOptions { + readonly url: string | URL +} + +const make = Effect.fnUntraced(function*(options: MakeOptions) { + type Service = typeof AdminApi.Service + + const auth = yield* Effect.serviceOption(Auth.Auth) + + const client = yield* HttpApiClient.make(AdminApiDefinition.Api, { + baseUrl: options.url, + transformClient: Option.match(auth, { + onNone: constUndefined, + onSome: (auth) => + HttpClient.mapRequestEffect( + Effect.fnUntraced(function*(request) { + const authInfo = yield* auth.getCachedAuthInfo + if (Option.isNone(authInfo)) return request + const token = authInfo.value.accessToken + return HttpClientRequest.bearerToken(request, token) + }) + ) + }) + }) + + // Dataset Operations + + const deployDataset: Service["deployDataset"] = Effect.fn("AdminApi.deployDataset")( + function*(namespace, name, revision, options) { + const path = { namespace, name, revision } + const payload = { + endBlock: options?.endBlock, + parallelism: options?.parallelism, + workerId: options?.workerId + } + yield* Effect.annotateCurrentSpan({ ...path, ...payload }) + return yield* client.dataset.deployDataset({ path, payload }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getDatasetManifest: Service["getDatasetManifest"] = Effect.fn("AdminApi.getDatasetManifest")( + function*(namespace, name, revision) { + const path = { namespace, name, revision } + yield* Effect.annotateCurrentSpan(path) + return yield* client.dataset.getDatasetManifest({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getDatasets: Service["getDatasets"] = Effect.fn("AdminApi.getDatasets")( + function*() { + return yield* client.dataset.getDatasets({}) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getDatasetVersion: Service["getDatasetVersion"] = Effect.fn("AdminApi.getDatasetVersion")( + function*(namespace, name, revision) { + const path = { namespace, name, revision } + yield* Effect.annotateCurrentSpan(path) + return yield* client.dataset.getDatasetVersion({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getDatasetVersions: Service["getDatasetVersions"] = Effect.fn("AdminApi.getDatasetVersions")( + function*(namespace, name) { + const path = { namespace, name } + yield* Effect.annotateCurrentSpan(path) + return yield* client.dataset.getDatasetVersions({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const registerDataset: Service["registerDataset"] = Effect.fn("AdminApi.registerDataset")( + function*(namespace, name, manifest, version) { + const payload = { namespace, name, version, manifest } + yield* Effect.annotateCurrentSpan(payload) + return yield* client.dataset.registerDataset({ payload }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getDatasetSyncProgress: Service["getDatasetSyncProgress"] = Effect.fn("AdminApi.getDatasetSyncProgress")( + function*(namespace, name, revision) { + const path = { namespace, name, revision } + yield* Effect.annotateCurrentSpan(path) + return yield* client.dataset.getDatasetSyncProgress({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + // Job Operations + + const getJobs: Service["getJobs"] = Effect.fn("AdminApi.getJobs")( + function*(options) { + const urlParams = { + limit: options?.limit, + lastJobId: options?.lastJobId, + status: options?.status + } + yield* Effect.annotateCurrentSpan(urlParams) + return yield* client.job.getJobs({ urlParams }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const getJobById: Service["getJobById"] = Effect.fn("AdminApi.getJobById")( + function*(jobId) { + const path = { jobId } + yield* Effect.annotateCurrentSpan(path) + return yield* client.job.getJobById({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const stopJob: Service["stopJob"] = Effect.fn("AdminApi.stopJob")( + function*(jobId) { + const path = { jobId } + yield* Effect.annotateCurrentSpan(path) + return yield* client.job.stopJob({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + const deleteJob: Service["deleteJob"] = Effect.fn("AdminApi.deleteJob")( + function*(jobId) { + const path = { jobId } + yield* Effect.annotateCurrentSpan(path) + return yield* client.job.deleteJob({ path }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + // Worker Operations + + const getWorkers: Service["getWorkers"] = Effect.fn("AdminApi.getWorkers")( + function*() { + return yield* client.worker.getWorkers({}) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + // Provider Operations + + const getProviders: Service["getProviders"] = Effect.fn("AdminApi.getProviders")( + function*() { + return yield* client.provider.getProviders({}) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + // Manifest Operations + + const registerManifest: Service["registerManifest"] = Effect.fn("AdminApi.registerManifest")( + function*(manifest) { + return yield* client.manifest.registerManifest({ payload: manifest }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + // Schema Operations + + const getOutputSchema: Service["getOutputSchema"] = Effect.fn("AdminApi.getOutputSchema")( + function*(payload) { + return yield* client.schema.getOutputSchema({ payload }) + }, + Effect.catchTag("HttpApiDecodeError", "ParseError", Effect.die) + ) + + return AdminApi.of({ + deployDataset, + getDatasetManifest, + getDatasets, + getDatasetVersion, + getDatasetVersions, + getDatasetSyncProgress, + registerDataset, + getJobs, + getJobById, + stopJob, + deleteJob, + getWorkers, + getProviders, + registerManifest, + getOutputSchema + }) +}) + +/** + * Creates a layer for the Admin API service. + */ +export const layer = (options: MakeOptions): Layer.Layer< + AdminApi, + never, + HttpClient.HttpClient +> => Layer.effect(AdminApi, make(options)) + +/** + * Creates a layer for the Admin API service with authentication provided by + * default. + */ +export const layerAuth = (options: MakeOptions): Layer.Layer< + AdminApi, + never, + HttpClient.HttpClient | KeyValueStore.KeyValueStore +> => + Layer.effect(AdminApi, make(options)).pipe( + Layer.provide(Auth.layer) + )