Add asynchronous data block processing

pydatalab/pyproject.toml

@@ -53,6 +53,7 @@ server = [
     "Flask-PyMongo ~= 2.3",
     "Flask-Mail ~= 0.10",
     "Flask-Compress ~= 1.15",
+    "APScheduler ~= 3.10",
     "Werkzeug ~= 3.0",
     "python-dotenv ~= 1.0",
     "pillow ~= 11.0",

pydatalab/schemas/collections.json

@@ -0,0 +1,363 @@
+{
+  "title": "Collection",
+  "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.",
+  "type": "object",
+  "properties": {
+    "blocks_obj": {
+      "title": "Blocks Obj",
+      "default": {},
+      "type": "object",
+      "additionalProperties": {
+        "$ref": "#/definitions/DataBlockResponse"
+      }
+    },
+    "display_order": {
+      "title": "Display Order",
+      "default": [],
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "creator_ids": {
+      "title": "Creator Ids",
+      "default": [],
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "creators": {
+      "title": "Creators",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/Person"
+      }
+    },
+    "type": {
+      "title": "Type",
+      "default": "collections",
+      "const": "collections",
+      "pattern": "^collections$",
+      "type": "string"
+    },
+    "immutable_id": {
+      "title": "Immutable ID",
+      "format": "uuid",
+      "type": "string"
+    },
+    "last_modified": {
+      "title": "Last Modified",
+      "type": "string",
+      "format": "date-time"
+    },
+    "relationships": {
+      "title": "Relationships",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/TypedRelationship"
+      }
+    },
+    "collection_id": {
+      "title": "Collection Id",
+      "minLength": 1,
+      "maxLength": 40,
+      "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$",
+      "type": "string"
+    },
+    "title": {
+      "title": "Title",
+      "type": "string"
+    },
+    "description": {
+      "title": "Description",
+      "type": "string"
+    },
+    "num_items": {
+      "title": "Num Items",
+      "type": "integer"
+    }
+  },
+  "definitions": {
+    "DataBlockResponse": {
+      "title": "DataBlockResponse",
+      "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.",
+      "type": "object",
+      "properties": {
+        "blocktype": {
+          "title": "Blocktype",
+          "type": "string"
+        },
+        "block_id": {
+          "title": "Block Id",
+          "type": "string"
+        },
+        "item_id": {
+          "title": "Item Id",
+          "type": "string"
+        },
+        "collection_id": {
+          "title": "Collection Id",
+          "type": "string"
+        },
+        "title": {
+          "title": "Title",
+          "type": "string"
+        },
+        "freeform_comment": {
+          "title": "Freeform Comment",
+          "type": "string"
+        },
+        "file_id": {
+          "title": "File Id",
+          "type": "string"
+        },
+        "file_ids": {
+          "title": "File Ids",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "errors": {
+          "title": "Errors",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "warnings": {
+          "title": "Warnings",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "b64_encoded_image": {
+          "title": "B64 Encoded Image",
+          "datalab_exclude_from_db": true,
+          "datalab_exclude_from_load": true,
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
+        "bokeh_plot_data": {
+          "title": "Bokeh Plot Data",
+          "datalab_exclude_from_db": true,
+          "datalab_exclude_from_load": true,
+          "type": "object"
+        },
+        "computed": {
+          "title": "Computed",
+          "datalab_exclude_from_load": true,
+          "type": "object"
+        },
+        "metadata": {
+          "title": "Metadata",
+          "datalab_exclude_from_load": true,
+          "type": "object"
+        }
+      },
+      "required": [
+        "blocktype",
+        "block_id"
+      ]
+    },
+    "RelationshipType": {
+      "title": "RelationshipType",
+      "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```",
+      "enum": [
+        "parent",
+        "child",
+        "sibling",
+        "is_part_of",
+        "other"
+      ],
+      "type": "string"
+    },
+    "KnownType": {
+      "title": "KnownType",
+      "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.",
+      "enum": [
+        "samples",
+        "starting_materials",
+        "blocks",
+        "files",
+        "people",
+        "collections"
+      ],
+      "type": "string"
+    },
+    "TypedRelationship": {
+      "title": "TypedRelationship",
+      "type": "object",
+      "properties": {
+        "description": {
+          "title": "Description",
+          "type": "string"
+        },
+        "relation": {
+          "$ref": "#/definitions/RelationshipType"
+        },
+        "type": {
+          "$ref": "#/definitions/KnownType"
+        },
+        "immutable_id": {
+          "title": "Immutable Id",
+          "type": "string"
+        },
+        "item_id": {
+          "title": "Item Id",
+          "minLength": 1,
+          "maxLength": 40,
+          "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$",
+          "type": "string"
+        },
+        "refcode": {
+          "title": "Refcode",
+          "minLength": 1,
+          "maxLength": 40,
+          "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type"
+      ]
+    },
+    "IdentityType": {
+      "title": "IdentityType",
+      "description": "A string enum representing the supported verifiable identity types.",
+      "enum": [
+        "email",
+        "orcid",
+        "github"
+      ],
+      "type": "string"
+    },
+    "Identity": {
+      "title": "Identity",
+      "description": "A model for identities that can be provided by external systems\nand associated with a given user.",
+      "type": "object",
+      "properties": {
+        "identity_type": {
+          "$ref": "#/definitions/IdentityType"
+        },
+        "identifier": {
+          "title": "Identifier",
+          "type": "string"
+        },
+        "name": {
+          "title": "Name",
+          "type": "string"
+        },
+        "verified": {
+          "title": "Verified",
+          "default": false,
+          "type": "boolean"
+        },
+        "display_name": {
+          "title": "Display Name",
+          "type": "string"
+        }
+      },
+      "required": [
+        "identity_type",
+        "identifier",
+        "name"
+      ]
+    },
+    "UserRole": {
+      "title": "UserRole",
+      "description": "An enumeration.",
+      "enum": [
+        "user",
+        "admin",
+        "manager"
+      ],
+      "type": "string"
+    },
+    "AccountStatus": {
+      "title": "AccountStatus",
+      "description": "A string enum representing the account status.",
+      "enum": [
+        "active",
+        "unverified",
+        "deactivated"
+      ],
+      "type": "string"
+    },
+    "Person": {
+      "title": "Person",
+      "description": "A model that describes an individual and their digital identities.",
+      "type": "object",
+      "properties": {
+        "type": {
+          "title": "Type",
+          "default": "people",
+          "const": "people",
+          "type": "string"
+        },
+        "immutable_id": {
+          "title": "Immutable ID",
+          "format": "uuid",
+          "type": "string"
+        },
+        "last_modified": {
+          "title": "Last Modified",
+          "type": "string",
+          "format": "date-time"
+        },
+        "relationships": {
+          "title": "Relationships",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TypedRelationship"
+          }
+        },
+        "identities": {
+          "title": "Identities",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/Identity"
+          }
+        },
+        "display_name": {
+          "title": "Display Name",
+          "minLength": 1,
+          "maxLength": 150,
+          "type": "string"
+        },
+        "contact_email": {
+          "title": "Contact Email",
+          "type": "string",
+          "format": "email"
+        },
+        "managers": {
+          "title": "Managers",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "role": {
+          "default": "user",
+          "allOf": [
+            {
+              "$ref": "#/definitions/UserRole"
+            }
+          ]
+        },
+        "account_status": {
+          "default": "unverified",
+          "allOf": [
+            {
+              "$ref": "#/definitions/AccountStatus"
+            }
+          ]
+        }
+      }
+    }
+  }
+}

pydatalab/src/pydatalab/apps/__init__.py

@@ -22,9 +22,10 @@ def load_app_blocks():
 
     try:
         # A dummy block that is used to check that bad blocks do not break the import
-        from pydatalab.apps._canary import CanaryBlock
+        from pydatalab.apps._canary import AsyncCanaryBlock, CanaryBlock
 
         app_blocks.append(CanaryBlock)
+        app_blocks.append(AsyncCanaryBlock)
     except ImportError as e:
         _check_error(e)