Update main to latest

.env.example

@@ -0,0 +1,14 @@
+DB_ENGINE=django.db.backends.postgresql
+DB_NAME=postgres
+DB_USER=postgres
+DB_PASSWORD=postgres
+DB_HOST=backend_db
+DB_PORT=5432
+TELEMETRY_URL=http://otel-collector:4317
+ELASTICSEARCH_INDEX=http://elasticsearch:9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASS=changeme
+URL_WHITELIST=http://localhost:8000,http://localhost,http://localhost:3000
+DEBUG=True
+SECRET_KEY=your-secret-key
+REDIS_URL=redis://redis:6379/1

.github/workflows/deploy-to-ecs.yml

@@ -0,0 +1,147 @@
+name: Deploy to Amazon ECS
+
+on:
+  push:
+    branches:
+      - dev
+  workflow_dispatch:
+
+env:
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }}
+  ECS_CLUSTER: ${{ secrets.ECS_CLUSTER }}
+  ECS_EXECUTION_ROLE_ARN: ${{ secrets.ECS_EXECUTION_ROLE_ARN }}
+  APP_NAME: dataspace
+  APP_PORT: 8000
+  DB_ENGINE: django.db.backends.postgresql
+  DB_PORT: 5432
+  DEBUG_MODE: "False"
+  TELEMETRY_URL: http://otel-collector:4317
+  CPU_UNITS: 256
+  MEMORY_UNITS: 512
+  SSM_PATH_PREFIX: /dataspace
+  ENVIRONMENT: ${{ secrets.ENVIRONMENT || 'dev' }}
+
+jobs:
+  deploy-infrastructure:
+    name: Deploy Infrastructure
+    runs-on: ubuntu-latest
+    environment: development
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.modified, 'aws/cloudformation')
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Deploy CloudFormation stack
+        run: |
+          aws cloudformation deploy \
+            --template-file aws/cloudformation/dataspace-infrastructure.yml \
+            --stack-name dataspace-${{ env.ENVIRONMENT }}-infrastructure \
+            --parameter-overrides \
+              Environment=${{ env.ENVIRONMENT }} \
+              VpcId=${{ secrets.VPC_ID }} \
+              SubnetIds=${{ secrets.SUBNET_IDS }} \
+              DBUsername=${{ secrets.DB_USERNAME }} \
+              DBPassword=${{ secrets.DB_PASSWORD }} \
+              DBName=${{ secrets.DB_NAME }} \
+              ElasticsearchPassword=${{ secrets.ELASTICSEARCH_PASSWORD }} \
+              DjangoSecretKey=${{ secrets.DJANGO_SECRET_KEY }} \
+            --capabilities CAPABILITY_IAM \
+            --no-fail-on-empty-changeset
+
+  deploy-app:
+    name: Deploy Application
+    runs-on: ubuntu-latest
+    environment: development
+    needs: deploy-infrastructure
+    if: always() # Run even if infrastructure deployment is skipped
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          IMAGE_TAG: ${{ github.sha }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Download task definition and get EFS ID
+        run: |
+          aws ecs describe-task-definition --task-definition dataspace --query taskDefinition > aws/current-task-definition.json
+          aws ecs describe-task-definition --task-definition dataspace-otel-collector --query taskDefinition > aws/current-otel-task-definition.json
+          # Get the EFS ID from CloudFormation export
+          EFS_ID=$(aws cloudformation list-exports --query "Exports[?Name=='dataspace-${{ env.ENVIRONMENT }}-MigrationsFileSystemId'].Value" --output text)
+          echo "EFS_ID=$EFS_ID" >> $GITHUB_ENV
+
+      - name: Update container image only
+        id: task-def-app
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: aws/current-task-definition.json
+          container-name: dataspace
+          image: ${{ steps.build-image.outputs.image }}
+
+      - name: Deploy main application ECS task definition
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.task-def-app.outputs.task-definition }}
+          service: ${{ secrets.ECS_SERVICE }}
+          cluster: ${{ env.ECS_CLUSTER }}
+          wait-for-service-stability: true
+
+  deploy-otel:
+    name: Deploy OpenTelemetry Collector
+    runs-on: ubuntu-latest
+    environment: development
+    needs: deploy-app
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Download current OpenTelemetry task definition
+        id: download-otel-taskdef
+        run: |
+          aws ecs describe-task-definition \
+            --task-definition dataspace-otel-collector \
+            --query taskDefinition > aws/current-otel-task-definition.json
+          cat aws/current-otel-task-definition.json
+
+      - name: Deploy OpenTelemetry ECS task definition
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: aws/current-otel-task-definition.json
+          service: ${{ secrets.ECS_OTEL_SERVICE }}
+          cluster: ${{ env.ECS_CLUSTER }}
+          wait-for-service-stability: true

.gitignore

@@ -155,3 +155,7 @@ resources/
 .env
 api/migrations/*
 authorization/migrations/*
+
+
+# AWS files
+aws/.env.*

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
+        exclude: ^aws/cloudformation/.*\.yml$
     -   id: check-added-large-files
     -   id: debug-statements
 
@@ -20,6 +21,17 @@ repos:
     -   id: isort
         args: ["--profile", "black"]
 
+-   repo: local
+    hooks:
+    -   id: cloudformation-validate
+        name: AWS CloudFormation Validation
+        description: Validates CloudFormation templates using AWS CLI
+        entry: bash -c 'aws cloudformation validate-template --template-body file://$0 || exit 1'
+        language: system
+        files: ^aws/cloudformation/.*\.yml$
+        require_serial: true
+        pass_filenames: true
+
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.9.0
     hooks:

Dockerfile

@@ -12,10 +12,18 @@ RUN echo 'deb http://archive.debian.org/debian stretch main contrib non-free' >>
 WORKDIR /code
 COPY . /code/
 
-RUN pip install psycopg2-binary
+RUN pip install psycopg2-binary uvicorn
 RUN pip install -r requirements.txt
-#RUN python manage.py migrate
+
+# Create healthcheck script
+RUN echo '#!/bin/bash\nset -e\npython -c "import sys; import django; django.setup(); sys.exit(0)"' > /code/healthcheck.sh \
+    && chmod +x /code/healthcheck.sh
 
 
 EXPOSE 8000
-#CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
+
+# Make entrypoint script executable
+RUN chmod +x /code/docker-entrypoint.sh
+
+ENTRYPOINT ["/code/docker-entrypoint.sh"]
+CMD ["uvicorn", "DataSpace.asgi:application", "--host", "0.0.0.0", "--port", "8000"]

api/activities/decorators.py

@@ -39,7 +39,7 @@ def decorator(func: F) -> F:
         def wrapper(*args: Any, **kwargs: Any) -> Any:
             # Extract request from args (typically the first or second argument in view functions)
             request = None
-            for arg in args:
+            for arg in list(args) + list(kwargs.values()):
                 if isinstance(arg, HttpRequest):
                     request = arg
                     break

api/models/UseCase.py

@@ -54,6 +54,7 @@ class UseCase(models.Model):
     )
     started_on = models.DateField(blank=True, null=True)
     completed_on = models.DateField(blank=True, null=True)
+    platform_url = models.URLField(blank=True, null=True)
 
     def save(self, *args: Any, **kwargs: Any) -> None:
         if self.title and not self.slug:

api/schema/dataset_schema.py

@@ -18,6 +18,7 @@
     ResourceChartDetails,
     ResourceChartImage,
     Sector,
+    UseCase,
 )
 from api.models.Dataset import Tag
 from api.models.DatasetMetadata import DatasetMetadata
@@ -31,7 +32,12 @@
 from api.types.type_organization import TypeOrganization
 from api.types.type_resource_chart import TypeResourceChart
 from api.types.type_resource_chart_image import TypeResourceChartImage
-from api.utils.enums import DatasetAccessType, DatasetLicense, DatasetStatus
+from api.utils.enums import (
+    DatasetAccessType,
+    DatasetLicense,
+    DatasetStatus,
+    UseCaseStatus,
+)
 from api.utils.graphql_telemetry import trace_resolver
 from authorization.models import DatasetPermission, OrganizationMembership, Role, User
 from authorization.permissions import (
@@ -469,20 +475,30 @@ def get_publishers(self, info: Info) -> List[Union[TypeOrganization, TypeUser]]:
         published_datasets = Dataset.objects.filter(
             status=DatasetStatus.PUBLISHED.value
         )
+        published_ds_organizations = published_datasets.values_list(
+            "organization_id", flat=True
+        )
+        published_usecases = UseCase.objects.filter(
+            status=UseCaseStatus.PUBLISHED.value
+        )
+        published_uc_organizations = published_usecases.values_list(
+            "organization_id", flat=True
+        )
+        published_organizations = set(published_ds_organizations) | set(
+            published_uc_organizations
+        )
 
         # Get unique organizations that have published datasets
         org_publishers = Organization.objects.filter(
-            id__in=published_datasets.filter(organization__isnull=False).values_list(
-                "organization_id", flat=True
-            )
+            id__in=published_organizations
         ).distinct()
 
+        published_ds_users = published_datasets.values_list("user_id", flat=True)
+        published_uc_users = published_usecases.values_list("user_id", flat=True)
+        published_users = set(published_ds_users) | set(published_uc_users)
+
         # Get unique individual users who have published datasets without an organization
-        individual_publishers = User.objects.filter(
-            id__in=published_datasets.filter(organization__isnull=True).values_list(
-                "user_id", flat=True
-            )
-        ).distinct()
+        individual_publishers = User.objects.filter(id__in=published_users).distinct()
 
         # Convert to GraphQL types
         org_types = [TypeOrganization.from_django(org) for org in org_publishers]
@@ -564,6 +580,10 @@ def add_update_dataset_metadata(
             dataset = Dataset.objects.get(id=dataset_id)
         except Dataset.DoesNotExist as e:
             raise DjangoValidationError(f"Dataset with ID {dataset_id} does not exist.")
+        if dataset.status != DatasetStatus.DRAFT.value:
+            raise DjangoValidationError(
+                f"Dataset with ID {dataset_id} is not in draft status."
+            )
 
         if update_metadata_input.description:
             dataset.description = update_metadata_input.description
@@ -616,11 +636,14 @@ def update_dataset(
             dataset = Dataset.objects.get(id=dataset_id)
         except Dataset.DoesNotExist as e:
             raise ValueError(f"Dataset with ID {dataset_id} does not exist.")
-
+        if dataset.status != DatasetStatus.DRAFT.value:
+            raise ValueError(f"Dataset with ID {dataset_id} is not in draft status.")
+        if update_dataset_input.title.strip() == "":
+            raise ValueError("Title cannot be empty.")
         if update_dataset_input.title:
-            dataset.title = update_dataset_input.title
+            dataset.title = update_dataset_input.title.strip()
         if update_dataset_input.description:
-            dataset.description = update_dataset_input.description
+            dataset.description = update_dataset_input.description.strip()
         if update_dataset_input.access_type:
             dataset.access_type = update_dataset_input.access_type
         if update_dataset_input.license:

api/schema/organization_data_schema.py

@@ -4,6 +4,7 @@
 
 import strawberry
 import strawberry_django
+from django.db.models import Q
 from strawberry.types import Info
 
 from api.models import Dataset, Organization, Sector, UseCase
@@ -58,7 +59,12 @@ def organization_published_use_cases(
         try:
             # Get published use cases for this organization
             queryset = UseCase.objects.filter(
-                usecaseorganizationrelationship__organization_id=organization_id,
+                (
+                    Q(organization__id=organization_id)
+                    | Q(
+                        usecaseorganizationrelationship__organization_id=organization_id
+                    )
+                ),
                 status=UseCaseStatus.PUBLISHED.value,
             ).distinct()
             return TypeUseCase.from_django_list(queryset)

api/schema/organization_schema.py

@@ -85,6 +85,15 @@ def organizations(
 
         return [TypeOrganization.from_django(org) for org in queryset]
 
+    @strawberry_django.field(permission_classes=[IsAuthenticated])
+    def all_organizations(self, info: Info) -> List[TypeOrganization]:
+        """Get all organizations."""
+        user = info.context.user
+        if not user or getattr(user, "is_anonymous", True):
+            logging.warning("Anonymous user or no user found in context")
+            return []
+        return [TypeOrganization.from_django(org) for org in Organization.objects.all()]
+
     @strawberry_django.field
     def organization(self, info: Info, id: str) -> Optional[TypeOrganization]:
         """Get organization by ID."""

api/schema/resource_schema.py

@@ -159,6 +159,11 @@ def _create_file_resource_schema(resource: Resource) -> None:
         return
 
 
+def _reset_file_resource_schema(resource: Resource) -> None:
+    ResourceSchema.objects.filter(resource=resource).delete()
+    data_table = index_resource_data(resource)
+
+
 def _update_file_resource_schema(
     resource: Resource, updated_schema: List[SchemaUpdate]
 ) -> None:
@@ -262,6 +267,7 @@ def create_file_resources(
                 file=file, size=file.size, resource=resource
             )
             _validate_file_details_and_update_format(resource)
+            _create_file_resource_schema(resource)
             resources.append(TypeResource.from_django(resource))
         return resources
 
@@ -351,6 +357,8 @@ def update_file_resource(
                     size=file_resource_input.file.size,
                     resource=resource,
                 )
+            _validate_file_details_and_update_format(resource)
+            _create_file_resource_schema(resource)
 
         if file_resource_input.preview_details:
             _update_resource_preview_details(file_resource_input, resource)