WIP - Task plugins

docs/Development.md

@@ -66,6 +66,17 @@ file_share_mounts:
 
 Instead of using the `file_share_mounts` setting, you can configure file share paths in the database. This is useful for production deployments where you want centralized management of file share paths. To use the paths in the database, set `file_share_mounts: []`. See [fileglancer-janelia](https://github.com/JaneliaSciComp/fileglancer-janelia) for an example of populating the file share paths in the database, using a private wiki source.
 
+### Making changes to the database
+If database changes are needed you first need to create a new alembic revision using:
+```
+pixi run migrate-create -m "enter what the migration is about"
+```
+Then run the actual up-migration
+```
+pixi run migrate
+```
+If database downgrade is required, so far there's no pixi task defined for it so that would have to be done in a `pixi shell`
+
 ### Running with SSL/HTTPS (Secure Mode)
 
 By default, `pixi run dev-launch` runs the server in insecure HTTP mode on port 7878. This is suitable for most local development scenarios.

fileglancer/alembic/versions/c05fd0a62deb_create_task_data_table.py

@@ -0,0 +1,45 @@
+"""create task data table
+
+Revision ID: c05fd0a62deb
+Revises: 9812335c52b6
+Create Date: 2025-11-12 10:53:48.450263
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c05fd0a62deb'
+down_revision = '9812335c52b6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tasks',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('owner', sa.String(), nullable=True),
+    sa.Column('proxy', sa.String(), nullable=True),
+    sa.Column('parameters', sa.String(), nullable=True),
+    sa.Column('compute_resources', sa.String(), nullable=True),
+    sa.Column('monitor_url', sa.String(), nullable=True),
+    sa.Column('output_log', sa.String(), nullable=True),
+    sa.Column('error_log', sa.String(), nullable=True),
+    sa.Column('status', sa.String(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('started_at', sa.DateTime(), nullable=True),
+    sa.Column('finished_at', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_tasks_name'), 'tasks', ['name'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_tasks_name'), table_name='tasks')
+    op.drop_table('tasks')
+    # ### end Alembic commands ###

fileglancer/app.py

@@ -7,7 +7,7 @@
 from datetime import datetime, timedelta, timezone, UTC
 from functools import cache
 from pathlib import Path as PathLib
-from typing import List, Optional, Dict, Tuple, Generator
+from typing import List, Optional, Dict, Tuple, Any
 
 try:
     import tomllib
@@ -34,6 +34,7 @@
 from fileglancer.utils import format_timestamp, guess_content_type, parse_range_header
 from fileglancer.user_context import UserContext, EffectiveUserContext, CurrentUserContext
 from fileglancer.filestore import Filestore
+from fileglancer.fgtasks.fgtasks import create_taskdata, get_tasks_registry
 from fileglancer.log import AccessLogMiddleware
 
 from x2s3.utils import get_read_access_acl, get_nosuchbucket_response, get_error_response
@@ -1088,6 +1089,56 @@ async def simple_login_handler(request: Request, body: dict = Body(...)):
 
         return response
 
+    @app.get("/api/tasks")
+    async def list_tasks():
+        tasks_registry = get_tasks_registry(get_settings())
+        # list tasks
+        task_names = tasks_registry.list_tasks()
+        return JSONResponse(content = task_names, status_code=200)
+
+    @app.get("/api/tasks/{task_name}/param-defs")
+    async def get_task_params(
+        task_name: str,
+        request: Request
+    ):
+        tasks_registry = get_tasks_registry(get_settings())
+        logger.info(f'Lookup task {task_name}')
+        # lookup task
+        task_defn = tasks_registry.get_task(task_name)
+        if task_defn is None:
+            logger.error(f'No task found for {task_name}')
+            return Response(status_code = 404)
+
+        # request.query_params is a MultiDict-like object
+        task_context = dict(request.query_params)
+
+        param_defs = [p.to_json() for p in task_defn.parameter_defns(task_context)]
+        return JSONResponse(content = param_defs, status_code=200)
+
+    @app.post("/api/tasks/{task_name}")
+    async def create_task(
+        task_name: str,
+        task_input: Dict[str, Any] = Body(...),
+        username: str = Depends(get_current_user)
+    ):
+        tasks_registry = get_tasks_registry(get_settings())
+        with _get_user_context(username):
+            # lookup task
+            task_defn = tasks_registry.get_task(task_name)
+
+            if task_defn is None:
+                logger.error(f'No task found for {task_name}')
+                return Response(status_code = 404)
+
+            # create and run task
+            task_data = create_taskdata(
+                task_name,
+                task_input.get('parameters',[]),
+                task_input.get('env', {}),
+                task_input.get('compute_resources', {}),
+            )
+            await task_defn.launch_task(task_data)
+            return Response(status_code=201)
 
     # Home page - redirect to /fg
     @app.get("/", include_in_schema=False)

fileglancer/auth.py

@@ -1,9 +1,8 @@
 """
 Authentication module for OKTA OAuth/OIDC integration
 """
-import os
 import hashlib
-from datetime import datetime, timedelta, UTC
+from datetime import datetime, UTC
 from typing import Optional
 
 from authlib.integrations.starlette_client import OAuth

fileglancer/cli.py

@@ -7,14 +7,13 @@
 import secrets
 import signal
 import sys
-import asyncio
 import click
 import uvicorn
 import json
 import webbrowser
 import threading
 import time
-import socket
+
 from pathlib import Path
 from datetime import datetime, timedelta, UTC
 from loguru import logger

fileglancer/database.py

@@ -137,6 +137,26 @@ class SessionDB(Base):
     last_accessed_at = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC))
 
 
+
+class TaskDataDB(Base):
+    """TaskData DB model"""
+    __tablename__ = 'tasks'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String, nullable=False, index=True)
+    owner = Column(String, nullable=True)
+    proxy = Column(String, nullable=True)
+    parameters = Column(String, nullable=True)
+    compute_resources = Column(String, nullable=True)
+    monitor_url = Column(String, nullable=True)
+    output_log = Column(String, nullable=True)
+    error_log = Column(String, nullable=True)
+    status = Column(String, nullable=False)
+    created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC))
+    started_at = Column(DateTime, nullable=True)
+    finished_at = Column(DateTime, nullable=True)
+
+
 def run_alembic_upgrade(db_url):
     """Run Alembic migrations to upgrade database to latest version"""
     global _migrations_run