Implement S3 media storage, event management, and API improvements

.env.example

@@ -21,15 +21,18 @@ GITHUB_TOKEN=
 
 # URLs written into exported metadata (DCAT / Croissant); set per environment
 PUBLIC_SITE_URL=https://civicdataspace.in
-PUBLIC_API_URL=https://api.civicdataspace.in
+PUBLIC_API_URL=https://api.datakeep.civicdays.in
 
-# --- s3 only ---
-# STORAGE_BUCKET=dataspace-dev
-# STORAGE_REGION=ap-south-1
-# STORAGE_ENV=dev
-# STORAGE_PUBLIC_BASE_URL=
-# STORAGE_ACCESS_KEY_ID=
-# STORAGE_SECRET_ACCESS_KEY=
+# Media storage (S3 only; required). STORAGE_ENV prefixes every key: use
+# "local" for local development against the dev bucket. Leave the access keys
+# empty on ECS/EC2 so the task role is used. STORAGE_PUBLIC_BASE_URL is the
+# base for public asset URLs (CDN or bucket URL); empty uses the S3 bucket URL.
+STORAGE_BUCKET=dataspace-dev
+STORAGE_REGION=ap-south-1
+STORAGE_ENV=local
+STORAGE_PUBLIC_BASE_URL=
+STORAGE_ACCESS_KEY_ID=
+STORAGE_SECRET_ACCESS_KEY=
 
 # Per-IP request throttling (api/middleware/rate_limit.py). False on dev only.
 RATE_LIMIT_ENABLED=True

.github/workflows/deploy-backend.yml

@@ -64,6 +64,11 @@ on:
         type: boolean
         required: false
         default: false
+      rollback:
+        description: "Roll dev back to the image that was live before the last deploy (no build, no tests). Ignored on main."
+        type: boolean
+        required: false
+        default: false
 
 # Queue overlapping deploys to the SAME environment rather than cancelling (a
 # cancelled run mid-deploy could leave .deploy/ state and the running
@@ -82,6 +87,7 @@ env:
 jobs:
   build:
     name: Build and push image
+    if: ${{ !inputs.rollback }}
     runs-on: ubuntu-latest
     timeout-minutes: 45
     permissions:
@@ -179,6 +185,9 @@ jobs:
               -e SECRET_KEY=ci-sanity-check-not-a-real-key \
               -e URL_WHITELIST=http://localhost \
               -e DB_ENGINE=django.db.backends.sqlite3 \
+              -e STORAGE_BUCKET=ci-sanity-check \
+              -e STORAGE_REGION=ap-south-1 \
+              -e STORAGE_ENV=ci \
               --entrypoint python \
               "${{ steps.ref.outputs.image_ref }}" \
               manage.py check; then
@@ -250,6 +259,9 @@ jobs:
   smoke-tests:
     name: Smoke Tests
     needs: deploy
+    # Repo variable SKIP_DEV_TESTS=true skips the dev test gates (e.g. while the
+    # test owner is away). main ignores it: prod deploys are always gated.
+    if: github.ref_name == 'main' || vars.SKIP_DEV_TESTS != 'true'
     # No `environment:` here -- GitHub rejects the entire workflow file at
     # parse time if a `uses:` job declares one. Consequence: this job also
     # cannot see environment-scoped vars, which is why api_base_url comes
@@ -262,7 +274,6 @@ jobs:
       # force_smoke_failure test path.
       deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }}
       min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }}
-      skip_provider: true
       # main: only the readonly allowlist ever runs against prod.
       # dev: stage 1 of 2 -- functional-tests runs only if this passes.
       suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }}
@@ -276,6 +287,12 @@ jobs:
       TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+      TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }}
+      TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }}
+      TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }}
+      TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }}
+      TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }}
+      TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }}
       # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a
       # confidential client, so without this the token request returns 401
       # and the job fails its preflight.
@@ -291,24 +308,41 @@ jobs:
     with:
       suite: functional
       api_base_url: ${{ vars.DEV_API_BASE_URL }}
-      skip_provider: true
     secrets:
       HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }}
       TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }}
       TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+      TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }}
+      TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }}
+      TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }}
+      TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }}
+      TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }}
+      TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }}
       KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}
 
+  tests-skipped:
+    name: Tests skipped (SKIP_DEV_TESTS)
+    needs: deploy
+    if: github.ref_name != 'main' && vars.SKIP_DEV_TESTS == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "::notice::SKIP_DEV_TESTS=true: smoke and functional tests were skipped for this dev deploy."
+          echo "**SKIP_DEV_TESTS=true**: smoke and functional tests were skipped. Delete the repo variable to turn them back on." >> "$GITHUB_STEP_SUMMARY"
+
   rollback-on-smoke-failure:
     name: Rollback (smoke tests failed)
     # `deploy` must be in needs: for needs.deploy.result to resolve here.
     needs: [deploy, smoke-tests, functional-tests]
     # Explicit results rather than failure(), so that a skipped functional stage
     # on main never reads as a failure. !cancelled() keeps cancellation a no-op.
+    # Also the manual rollback button: Run workflow on dev with `rollback` ticked.
     if: >-
-      ${{ !cancelled() && needs.deploy.result == 'success' &&
-      (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }}
+      ${{ !cancelled() && ((inputs.rollback && github.ref_name != 'main') ||
+      (needs.deploy.result == 'success' &&
+      (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure'))) }}
     runs-on: ubuntu-latest
     environment: ${{ github.ref_name == 'main' && 'production' || 'development' }}
     # 50m: must comfortably exceed the Restore previous image step's own
@@ -347,6 +381,7 @@ jobs:
           script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}"
 
       - name: Mark this run as failed
+        if: ${{ !inputs.rollback }}
         # The mitigation succeeded, but the run must still read RED -- a bad
         # deploy that silently self-heals is a bad deploy nobody investigates.
         run: |
@@ -357,10 +392,10 @@ jobs:
     name: Finalize Deploy
     needs: [deploy, smoke-tests, functional-tests]
     # Explicit results, not success()/!failure(): on main the functional stage
-    # is skipped by design.
+    # is skipped by design, and on dev SKIP_DEV_TESTS skips smoke too.
     if: >-
       ${{ !cancelled() && needs.deploy.result == 'success' &&
-      needs.smoke-tests.result == 'success' &&
+      (needs.smoke-tests.result == 'success' || needs.smoke-tests.result == 'skipped') &&
       (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }}
     runs-on: ubuntu-latest
     environment: ${{ github.ref_name == 'main' && 'production' || 'development' }}

.github/workflows/deploy-to-ecs.yml

@@ -158,6 +158,12 @@ jobs:
       TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+      TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }}
+      TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }}
+      TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }}
+      TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }}
+      TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }}
+      TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }}
       # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a
       # confidential client, so without this the token request returns 401
       # and the job fails its preflight.

.github/workflows/django-tests.yml

@@ -16,6 +16,20 @@ jobs:
   django-tests:
     runs-on: ubuntu-latest
 
+    # The data database: tests marked data_db build real tables in it.
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
     steps:
       - uses: actions/checkout@v4
 
@@ -40,6 +54,10 @@ jobs:
       # tests/object_types/charts and the dataset metadata tests were left
       # behind by the chart consolidation and fail on main: see #189.
       - name: Run tests
+        env:
+          TEST_DATA_DB_URL: postgres://postgres:postgres@localhost:5432/postgres
+          # Fail the run instead of skipping the data_db tests if the database is missing.
+          REQUIRE_DATA_DB_TESTS: "1"
         run: |
           pytest tests/ \
             --ignore=tests/object_types/charts \

.github/workflows/pr-gate.yml

@@ -48,4 +48,10 @@ jobs:
       TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+      TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }}
+      TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }}
+      TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }}
+      TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }}
+      TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }}
+      TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }}
       KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}

.github/workflows/sdk-tests.yml

@@ -8,36 +8,37 @@
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # Pinned: ubuntu-latest (26.04) has no Python 3.9 build, which the SDK still supports.
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[dev]"
 
       - name: Run type checking with mypy
         run: |
           mypy dataspace_sdk/ --ignore-missing-imports --config-file=/dev/null
 
       - name: Run tests with pytest
         run: |
           pytest tests/test_auth.py tests/test_base.py tests/test_client.py tests/test_datasets.py tests/test_aimodels.py tests/test_usecases.py tests/test_exceptions.py -v -p no:django -o addopts= --override-ini=django_find_project=false --noconftest --cov=dataspace_sdk --cov-report=term-missing --cov-report=xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           file: ./coverage.xml
           flags: unittests
           name: codecov-umbrella
           fail_ci_if_error: false