From 572a56a3f6bc1dd05ecb29663400bcfff0135b6d Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 9 Oct 2026 19:07:52 +0530 Subject: [PATCH 01/55] ci(deploy): SKIP_DEV_TESTS repo variable skips the dev test gates A green 'Tests skipped' job records it on the run, and Finalize Deploy still prunes images. main ignores the flag. --- .github/workflows/deploy-backend.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 85158cec..6dd90bae 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -250,6 +250,9 @@ jobs: smoke-tests: name: Smoke Tests needs: deploy + # Repo variable SKIP_DEV_TESTS=true skips the dev test gates (e.g. while the + # test owner is away). main ignores it: prod deploys are always gated. + if: github.ref_name == 'main' || vars.SKIP_DEV_TESTS != 'true' # No `environment:` here -- GitHub rejects the entire workflow file at # parse time if a `uses:` job declares one. Consequence: this job also # cannot see environment-scoped vars, which is why api_base_url comes @@ -300,6 +303,16 @@ jobs: TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + tests-skipped: + name: Tests skipped (SKIP_DEV_TESTS) + needs: deploy + if: github.ref_name != 'main' && vars.SKIP_DEV_TESTS == 'true' + runs-on: ubuntu-latest + steps: + - run: | + echo "::notice::SKIP_DEV_TESTS=true: smoke and functional tests were skipped for this dev deploy." + echo "**SKIP_DEV_TESTS=true**: smoke and functional tests were skipped. Delete the repo variable to turn them back on." >> "$GITHUB_STEP_SUMMARY" + rollback-on-smoke-failure: name: Rollback (smoke tests failed) # `deploy` must be in needs: for needs.deploy.result to resolve here. @@ -357,10 +370,10 @@ jobs: name: Finalize Deploy needs: [deploy, smoke-tests, functional-tests] # Explicit results, not success()/!failure(): on main the functional stage - # is skipped by design. + # is skipped by design, and on dev SKIP_DEV_TESTS skips smoke too. if: >- ${{ !cancelled() && needs.deploy.result == 'success' && - needs.smoke-tests.result == 'success' && + (needs.smoke-tests.result == 'success' || needs.smoke-tests.result == 'skipped') && (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} From 2f5f92525f1f45e0be88bae89b1f8d6e7f64fb74 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 9 Oct 2026 19:08:30 +0530 Subject: [PATCH 02/55] ci(deploy): manual rollback button for dev Run workflow on dev with 'rollback' ticked skips the build and runs the existing rollback job, which restores the previously live image. The run stays green when the rollback succeeds. --- .github/workflows/deploy-backend.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 6dd90bae..ce870818 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -64,6 +64,11 @@ on: type: boolean required: false default: false + rollback: + description: "Roll dev back to the image that was live before the last deploy (no build, no tests). Ignored on main." + type: boolean + required: false + default: false # Queue overlapping deploys to the SAME environment rather than cancelling (a # cancelled run mid-deploy could leave .deploy/ state and the running @@ -82,6 +87,7 @@ env: jobs: build: name: Build and push image + if: ${{ !inputs.rollback }} runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -319,9 +325,11 @@ jobs: needs: [deploy, smoke-tests, functional-tests] # Explicit results rather than failure(), so that a skipped functional stage # on main never reads as a failure. !cancelled() keeps cancellation a no-op. + # Also the manual rollback button: Run workflow on dev with `rollback` ticked. if: >- - ${{ !cancelled() && needs.deploy.result == 'success' && - (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }} + ${{ !cancelled() && ((inputs.rollback && github.ref_name != 'main') || + (needs.deploy.result == 'success' && + (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure'))) }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} # 50m: must comfortably exceed the Restore previous image step's own @@ -360,6 +368,7 @@ jobs: script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" - name: Mark this run as failed + if: ${{ !inputs.rollback }} # The mitigation succeeded, but the run must still read RED -- a bad # deploy that silently self-heals is a bad deploy nobody investigates. run: | From df34e5a0f8d27e495394c92dcbc5b088003ab3b9 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 12 Oct 2026 20:27:26 +0530 Subject: [PATCH 03/55] fix(api): point PUBLIC_API_URL fallback at the live prod API The fallback https://api.civicdataspace.in does not resolve, so every download_url in DCAT/Croissant/Dublin Core exports was a dead link when PUBLIC_API_URL was unset. Use https://api.datakeep.civicdays.in, the prod API host, in settings, the export adapter and .env.example. Closes #238 --- .env.example | 2 +- DataSpace/settings.py | 2 +- api/services/metadata_export/adapter.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 7887793e..136af89e 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ GITHUB_TOKEN= # URLs written into exported metadata (DCAT / Croissant); set per environment PUBLIC_SITE_URL=https://civicdataspace.in -PUBLIC_API_URL=https://api.civicdataspace.in +PUBLIC_API_URL=https://api.datakeep.civicdays.in # --- s3 only --- # STORAGE_BUCKET=dataspace-dev diff --git a/DataSpace/settings.py b/DataSpace/settings.py index f3e4c0ab..d390827e 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -306,7 +306,7 @@ # Absolute URLs written into exported metadata documents (dataset landing page, # download links). Override on any environment that is not production. PUBLIC_SITE_URL = os.getenv("PUBLIC_SITE_URL", "https://civicdataspace.in") -PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.civicdataspace.in") +PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.datakeep.civicdays.in") # DVC settings DVC_REPO_PATH = os.path.join(BASE_DIR, "dvc") diff --git a/api/services/metadata_export/adapter.py b/api/services/metadata_export/adapter.py index 2899c71b..9c8cfcda 100644 --- a/api/services/metadata_export/adapter.py +++ b/api/services/metadata_export/adapter.py @@ -25,7 +25,7 @@ def _public_base() -> str: def _api_base() -> str: - return getattr(settings, "PUBLIC_API_URL", "https://api.civicdataspace.in").rstrip("/") + return getattr(settings, "PUBLIC_API_URL", "https://api.datakeep.civicdays.in").rstrip("/") def _iso(dt: Any) -> Optional[str]: From b66f511658840a17d8c8620a00115e10f5aada73 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 12 Oct 2026 21:38:32 +0530 Subject: [PATCH 04/55] fix(api): stop returning exception text from metadata export The export view caught every ValueError around export_dataset() and sent str(exc) back with a 400, so internal errors (crosswalk, dates, rdflib) reached anonymous users verbatim and were reported as client errors. Validate standard and format against export_options() before exporting and answer with a fixed message plus the options. Every other failure now falls to the logged generic 500. Closes #240 --- api/views/metadata_export.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/api/views/metadata_export.py b/api/views/metadata_export.py index a3df43cb..bb7bf5c3 100644 --- a/api/views/metadata_export.py +++ b/api/views/metadata_export.py @@ -43,11 +43,18 @@ def metadata_export(request: HttpRequest, dataset_id: uuid.UUID) -> HttpResponse standard = (request.GET.get("standard") or "dcat").strip().lower() fmt = (request.GET.get("format") or "jsonld").strip().lower() + # Only these two inputs are the caller's fault. Fixed messages: never echo + # an exception to a public client. + options = export_options() + if standard not in options: + return JsonResponse({"error": "Unknown standard", "options": options}, status=400) + if fmt not in options[standard]["formats"]: + return JsonResponse( + {"error": "Format not available for this standard", "options": options}, status=400 + ) try: body, content_type, ext, report = export_dataset(dataset, standard, fmt) - except ValueError as exc: - return JsonResponse({"error": str(exc), "options": export_options()}, status=400) - except Exception as exc: # pragma: no cover - defensive; never 500 on a public page + except Exception as exc: # internal failure: log it, return a fixed message logger.error( "metadata_export_failed", dataset_id=str(dataset_id), standard=standard, error=str(exc) ) From 0aed4880ff3136ec92f8a86c9e1d204a2be4e474 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Tue, 13 Oct 2026 00:44:03 +0530 Subject: [PATCH 05/55] add network override for docker hotreload --- docker-compose.hotreload.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.hotreload.yml b/docker-compose.hotreload.yml index d8fce17a..a25508f5 100644 --- a/docker-compose.hotreload.yml +++ b/docker-compose.hotreload.yml @@ -7,3 +7,6 @@ services: command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload volumes: - .:/code + + networks: !override + - default \ No newline at end of file From 5a5bc26931811e3f34249ef45b673139deaa66a3 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Tue, 13 Oct 2026 00:44:49 +0530 Subject: [PATCH 06/55] add AGENTS.md --- AGENTS.md | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..78f32026 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,178 @@ +# AGENTS.md + +You are a senior engineer on the DataSpace backend. + +## Persona +- You write clean, reusable, maintainable code that reads like the code around it. +- You understand the codebase and its security risks (authorization, multi-tenant organization data, file uploads, search exposure), and you turn that into clear, actionable insights. +- Your output: small, well-tested changes and reviews that a teammate can merge with confidence, plus a short explanation of *why* when the reason isn't obvious. + +## Commands +- **Run locally (Docker, hot reload):** `docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build` +- **Run locally (venv):** copy `.env.example` to `.env`, then `python manage.py runserver` +- **Test:** `pytest tests/` uses SQLite in-memory with `--nomigrations` (see `pytest.ini`). It needs `pytest pytest-django`, which are not in `requirements.txt`. CI runs it with `--ignore=tests/object_types/charts --deselect tests/schema/test_dataset_schema.py::TestAddUpdateDatasetMetadata` (known-broken, see #189). +- **Single test:** `pytest tests/test_slug_collision.py -k usecase` +- **Format:** `black .` and `isort --profile black .` (line length 100, set in `pyproject.toml`; that file is the SDK's packaging config, but black reads it for the whole repo) +- **Type check:** `mypy --config-file=mypy.ini .` +- **All hooks:** `pre-commit run --all-files` (whitespace, black, isort, mypy, `debug-statements`) +- **Migrations:** `python manage.py makemigrations api`, then `python manage.py migrate` +- **Search mappings:** `python manage.py sync_search_mappings` after changing anything in `search/documents/` +- **API testing:** Postman CLI is installed; collections live in `postman/collections/` + +## Definition of done +Before you call a change finished: +1. `pre-commit run --all-files` is clean. +2. The tests pass with the CI flags above, and a bug fix has a regression test. +3. Every new or changed GraphQL query/mutation has the right permission guard (see Rules). +4. If a model changed: its search document, signals, cache invalidation and SDK still match. + +## Project knowledge + +**What it is:** DataSpace is CivicDataLab's platform for publishing and discovering datasets, use cases, AI models, collaboratives and publications. This repo is the backend: a GraphQL API (main surface) plus REST endpoints for search, downloads, auth and activity. It also ships a Python client, `dataspace_sdk`, published to PyPI. + +**Tech stack:** +- Python 3.11, Django 5.0, PostgreSQL +- GraphQL: `strawberry-graphql` 0.235 + `strawberry-graphql-django` 0.42 (endpoint `/api/graphql`) +- REST: Django REST Framework 3.14, `drf-yasg` for API docs +- Auth: Keycloak (`python-keycloak` 5.5); every request needs a real Keycloak JWT, with no dev-mode fallback +- Search: Elasticsearch 8 via `django-elasticsearch-dsl` +- Cache: Redis (`django-redis`, `django-cacheops`) +- Activity feed: `django-activity-stream` +- Observability: OpenTelemetry, `structlog` +- Data: pandas, pyarrow, DVC for dataset versioning +- Runtime: uvicorn/gunicorn in Docker, deployed to AWS (ECS / EC2 via `scripts/ci-*.sh`) + +**File structure:** +- `DataSpace/`: Django project (`settings.py`, `urls.py`, `asgi.py`, `cache_settings.py`) +- `api/`: main app + - `models/`: one model per file (`Dataset.py`, `Organization.py`, `UseCase.py`, …), exported from `models/__init__.py` + - `schema/`: Strawberry `Query`/`Mutation` classes per domain (`*_schema.py`), stitched together in `schema/schema.py`. `base_mutation.py` holds `BaseMutation`/`MutationResponse`. `extensions.py` holds `TrackActivity`/`TrackModelActivity`. + - `types/`: Strawberry output types (`type_*.py`) + - `views/`: REST views (search, downloads, charts, health, auth) + - `services/`: business logic kept out of resolvers (publication, platform import, metadata mapping/export, model clients) + - `activities/`: activity-stream handlers and display per domain (`dataset.py`, `organization.py`, …), wired up in `registry.py` + - `managers/`: `dvc_manager.py` for dataset versioning + - `signals/`: post-save hooks (search indexing, cache invalidation) + - `middleware/`, `utils/middleware.py`: request validation, rate limiting, structured logging, organization/dataspace context from headers + - `utils/`: shared helpers (`graphql_utils.py`, `keycloak_utils.py`, `search_query.py`, `search_cache.py`, `enums.py`, …) + - `management/commands/`: `sync_search_mappings`, `update_dataindex`, `seed_resource_types`, `populate_geographies`, … + - `migrations/`: numbered sequentially (`00NN_*`) +- `authorization/`: Keycloak integration, roles, `OrganizationMembership`, consent/activity middleware + - `permissions.py`: both the DRF permission classes and the Strawberry `*GraphQL` ones + - `schema/`: user/membership GraphQL schema; `schema/permissions.py` holds `HasOrganizationAdminRole` + - `graphql_permissions.py`: legacy duplicate of the GraphQL classes; don't add new uses +- `search/documents/`: Elasticsearch document definitions and analysers +- `dataspace_sdk/`: standalone Python SDK (its own version in `pyproject.toml`; depends only on `requests`) +- `tests/`: pytest suite (`test_*.py`, `tests/schema/`, `tests/journeys/`), settings in `tests/test_settings.py` +- `docs/`: feature docs (Keycloak, unified search, charts, activity stream, SDK) +- `postman/`: Postman collections and fixtures for API testing +- `aws/`, `scripts/`, `.github/workflows/`: infra, deploy scripts, CI +- Root-level `migrate_*.py`, `populate_sdgs.py`, `test_geography_hierarchy.py`: one-off scripts. Leave them alone, and put new code in `api/management/commands/` instead. + +## Standards + +**Naming conventions:** +- Functions, methods, variables: snake_case (`create_organization`, `get_current_user`) +- Classes: PascalCase (`OrganizationInput`, `HasOrganizationRole`) +- Constants: UPPER_SNAKE_CASE (`MAX_RETRIES`) +- Strawberry output types: `Type` in `api/types/type_.py` +- Model files: PascalCase, one model per file (`api/models/UseCase.py`) +- GraphQL fields are snake_case in Python. Strawberry exposes them as camelCase (`create_organization` → `createOrganization`). + +**Rules:** +- Type-annotate every function. mypy runs in pre-commit, so don't add new `# type: ignore` or `mypy.ini` exemptions without a reason in a comment. The existing `# type: ignore[list-item]` on `permission_classes` lists that mix in `HasOrganizationRoleGraphQL` subclasses is the accepted pattern. + - mypy skips `tests/` and is relaxed for `api.models`, `api.activities` and the resource, use case and dataset schemas (see `mypy.ini`). Still annotate new code there. +- Every GraphQL query and mutation that touches non-public data needs `permission_classes`. Don't rely on the ORM to reject anonymous users. + - Import from `authorization.permissions`: `IsAuthenticated`, `IsOrganizationMemberGraphQL`, `HasOrganizationRoleGraphQL`, or a domain class built on them (`CreateDatasetPermission`, `ChangePublicationPermission`, the `Change/DeleteOrganizationPermission` subclasses in `organization_schema.py`, …). + - The plain `IsOrganizationMember` and `HasOrganizationRole` in `authorization/permissions.py` are DRF classes for REST views. They don't work in Strawberry `permission_classes`. Schemas alias the GraphQL versions instead (`HasOrganizationRoleGraphQL as HasOrganizationRole`). + - `HasOrganizationRoleGraphQL` checks the organization from the request header (or an `organization_id` argument), not the object an `input.id` points at. When a mutation loads an object by id, also check the caller's membership in that object's organization inside the resolver. +- Scope querysets to what the user can see (organization membership, dataspace, published status). Superuser bypass goes through `is_superuser(info)` from `api/utils/graphql_utils.py`. +- Keep resolvers thin. Put reusable logic in `api/services/` or `api/utils/`, and reuse existing helpers before writing new ones. +- Wrap multi-step writes in `transaction.atomic()`. +- Return clean client errors (400/404, or `MutationResponse.error_response`) instead of letting `IntegrityError`, `ValueError` or `DoesNotExist` surface as a 500. +- Log with the existing logger and `structlog`, without tokens or PII. +- Comments explain *why* (constraints, past incidents), not *what*. Match the density of the surrounding file. +- Write regression tests in the style of `tests/test_slug_collision.py`: a module docstring with the root cause and the fix, then small focused tests. + +**Code style example:** +```python +from api.schema.extensions import TrackModelActivity +from api.utils.graphql_utils import is_superuser +from authorization.models import OrganizationMembership +from authorization.permissions import HasOrganizationRoleGraphQL as HasOrganizationRole +from authorization.permissions import IsAuthenticated + + +class ChangeOrganizationPermission(HasOrganizationRole): + def __init__(self) -> None: + super().__init__(operation="change") + + +# ✅ Good - permission guard, typed, scoped, atomic, clean error +@strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[IsAuthenticated, ChangeOrganizationPermission], # type: ignore[list-item] + extensions=[ + TrackModelActivity( + verb="updated", + get_data=lambda result, **kwargs: { + "organization_name": result.name, + "organization_id": str(result.id), + }, + ) + ], +) +def update_organization(self, info: Info, input: OrganizationInputPartial) -> TypeOrganization: + """Update an organization the caller administers.""" + try: + organization = Organization.objects.get(id=input.id) + except Organization.DoesNotExist: + raise ValueError(f"Organization with ID {input.id} does not exist.") + + # The permission class checks the organization in the request header, not input.id + is_admin = OrganizationMembership.objects.filter( + user=info.context.user, organization=organization, role__name="admin" + ).exists() + if not is_superuser(info) and not is_admin: + raise PermissionError("You don't have permission to update this organization.") + + with transaction.atomic(): + for field, value in vars(input).items(): + if field != "id" and value is not strawberry.UNSET: + setattr(organization, field, value) + organization.save() + return TypeOrganization.from_django(organization) + + +# ❌ Bad - no auth guard, untyped, unscoped lookup, raw 500 on a missing id +@strawberry_django.mutation +def update_org(self, info, input): + org = Organization.objects.get(id=input.id) + for k, v in vars(input).items(): + setattr(org, k, v) + org.save() + return org +``` + +## Git & PRs +- Branch from `dev` with `fix/…`, `feat/…` or `chore/…`. PRs go into `dev`. Releases go from `dev` to `main`, gated by `pr-gate.yml`. +- Do not make any commits utill asked for it. +- Use Conventional Commits with a scope: `fix(api): …`, `feat(usecase): …`. +- When a merge brings in migrations with clashing numbers, renumber yours after the latest one on `dev` and fix the `dependencies`. + +## Boundaries +- ✅ **Always:** write code in `api/`, `authorization/`, `search/`, `dataspace_sdk/` and `tests/`. Run `pytest` and `pre-commit` before committing. +- ⚠️ **Ask first:** + - model or schema changes and new migrations + - changes to permission classes, Keycloak/auth middleware or rate limits + - changes to Elasticsearch mappings (these need a reindex) + - adding or upgrading dependencies + - bumping the SDK version or publishing it + - touching `.github/workflows/`, `aws/`, `scripts/ci-*.sh`, `Dockerfile` or `docker-compose.yml` + - anything that runs against a deployed environment +- 🚫 **Never:** + - commit `.env`, secrets, Keycloak client secrets or tokens. Use `.env.example` for new variables. + - edit or delete already-merged migrations. Add a new migration instead. + - disable an auth check or a failing test to get green. Fix the cause, or ask. + - edit `.venv/`, `files/` (user uploads) or `logs/` + - force-push to `dev` or `main` From e4c52db4d859a2ec67bd865da410ec1ee88d0cd9 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:10:37 +0530 Subject: [PATCH 07/55] refactor(api)!: remove DVC dataset versioning DVC never worked: uploaded files were never under DVC_REPO_PATH, so every DVC call raised and createMajorVersion always failed. Remove the manager, the setup command, version-change detection and the post_save signal. createMajorVersion now writes the ResourceVersion row directly inside transaction.atomic(), behind IsAuthenticated plus a check that the caller owns the dataset or may change it. BREAKING CHANGE: the setup_dvc management command and DVC_* settings are gone. --- DataSpace/settings.py | 5 - .../commands/backfill_file_hashes.py | 2 +- api/management/commands/setup_dvc.py | 131 ------- api/managers/dvc_manager.py | 132 ------- api/models/Resource.py | 162 +-------- api/schema/resource_schema.py | 63 ++-- api/signals/dataset_signals.py | 22 +- api/utils/version_detection.py | 327 ------------------ docs/version_detection_system.md | 140 -------- requirements.txt | 1 - 10 files changed, 28 insertions(+), 957 deletions(-) delete mode 100644 api/management/commands/setup_dvc.py delete mode 100644 api/managers/dvc_manager.py delete mode 100644 api/utils/version_detection.py delete mode 100644 docs/version_detection_system.md diff --git a/DataSpace/settings.py b/DataSpace/settings.py index d390827e..63452733 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -308,11 +308,6 @@ PUBLIC_SITE_URL = os.getenv("PUBLIC_SITE_URL", "https://civicdataspace.in") PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.datakeep.civicdays.in") -# DVC settings -DVC_REPO_PATH = os.path.join(BASE_DIR, "dvc") -DVC_REMOTE_NAME = os.getenv("DVC_REMOTE_NAME", None) -DVC_REMOTE_URL = os.getenv("DVC_REMOTE_URL", None) - # Django REST Framework settings REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": [ diff --git a/api/management/commands/backfill_file_hashes.py b/api/management/commands/backfill_file_hashes.py index 139af5dd..a97e5172 100644 --- a/api/management/commands/backfill_file_hashes.py +++ b/api/management/commands/backfill_file_hashes.py @@ -3,7 +3,7 @@ python manage.py backfill_file_hashes # fill every missing hash python manage.py backfill_file_hashes --dry-run # only count -Writes through a queryset update, so the DVC versioning signal does not run. +Writes through a queryset update, so no model save signals run. """ from django.core.management.base import BaseCommand diff --git a/api/management/commands/setup_dvc.py b/api/management/commands/setup_dvc.py deleted file mode 100644 index 22808721..00000000 --- a/api/management/commands/setup_dvc.py +++ /dev/null @@ -1,131 +0,0 @@ -import os -import subprocess -from typing import Any, Optional, cast - -import structlog -from django.conf import settings -from django.core.management.base import BaseCommand - -logger = structlog.getLogger(__name__) - - -class Command(BaseCommand): - help = "Set up DVC repository for dataset versioning" - - def handle(self, *args: Any, **options: Any) -> None: - try: - repo_path = settings.DVC_REPO_PATH - - # Create directory if needed - if not os.path.exists(repo_path): - os.makedirs(repo_path) - self.stdout.write(f"Created DVC repository directory at {repo_path}") - - # Initialize Git and DVC if not already done - if not os.path.exists(os.path.join(repo_path, ".git")): - subprocess.run(["git", "init"], cwd=repo_path, check=True) - self.stdout.write("Initialized Git repository") - - if not os.path.exists(os.path.join(repo_path, ".dvc")): - subprocess.run(["dvc", "init"], cwd=repo_path, check=True) - self.stdout.write("Initialized DVC repository") - - # Configure chunking for large files - subprocess.run( - ["dvc", "config", "cache.type", "hardlink,symlink"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["dvc", "config", "cache.shared", "group"], - cwd=repo_path, - check=True, - ) - - # Configure cache size limits to prevent excessive disk usage - subprocess.run( - ["dvc", "config", "cache.size_limit", "10G"], - cwd=repo_path, - check=True, - ) - - self.stdout.write("Configured DVC for large file handling") - - # Set up remote if configured - if ( - hasattr(settings, "DVC_REMOTE_NAME") - and hasattr(settings, "DVC_REMOTE_URL") - and settings.DVC_REMOTE_NAME - and settings.DVC_REMOTE_URL - ): - # Cast to str to satisfy mypy - remote_name = cast(str, settings.DVC_REMOTE_NAME) - remote_url = cast(str, settings.DVC_REMOTE_URL) - - subprocess.run( - [ - "dvc", - "remote", - "add", - remote_name, - remote_url, - ], - cwd=repo_path, - check=True, - ) - - # Set as default remote - subprocess.run( - ["dvc", "remote", "default", remote_name], - cwd=repo_path, - check=True, - ) - - self.stdout.write(f"Configured DVC remote: {remote_name}") - - # Set up Git user if not already configured - try: - # Check if Git user is configured - result = subprocess.run( - ["git", "config", "user.name"], - cwd=repo_path, - capture_output=True, - text=True, - ) - if not result.stdout.strip(): - subprocess.run( - ["git", "config", "user.name", "DataEx Bot"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "dataex@example.com"], - cwd=repo_path, - check=True, - ) - self.stdout.write("Configured Git user for DVC repository") - except subprocess.CalledProcessError: - # If checking fails, set the user anyway - subprocess.run( - ["git", "config", "user.name", "DataEx Bot"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "dataex@example.com"], - cwd=repo_path, - check=True, - ) - self.stdout.write("Configured Git user for DVC repository") - - self.stdout.write(self.style.SUCCESS("DVC repository set up successfully")) - - except Exception as e: - import traceback - - logger.error( - f"Failed to set up DVC repository: {str(e)}, Traceback: {traceback.format_exc()}" - ) - self.stdout.write( - self.style.ERROR(f"Failed to set up DVC repository: {str(e)}") - ) diff --git a/api/managers/dvc_manager.py b/api/managers/dvc_manager.py deleted file mode 100644 index e9c98195..00000000 --- a/api/managers/dvc_manager.py +++ /dev/null @@ -1,132 +0,0 @@ -import subprocess -from pathlib import Path -from typing import List, Union - -import structlog - -logger = structlog.getLogger(__name__) - - -class DVCManager: - def __init__(self, repo_path: Union[str, Path]) -> None: - self.repo_path = Path(repo_path) - - def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: - try: - return subprocess.run( - command, - cwd=str(self.repo_path), - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - # Log the error with details - logger.error(f"Command failed: {' '.join(command)}, Error: {e.stderr}") - # You can choose to re-raise or handle differently - raise - - def track_resource(self, file_path: Union[str, Path], chunked: bool = False) -> str: - """Add a resource file to DVC tracking with optional chunking for large files""" - rel_path = Path(file_path).relative_to(self.repo_path) - cmd = ["dvc", "add"] - if chunked: - cmd.append("--chunked") # Enables chunking for large files - cmd.append(str(rel_path)) - self._run_command(cmd) - return str(rel_path) + ".dvc" - - def commit_version(self, dvc_file: str, message: str) -> None: - """Commit the DVC file to git""" - self._run_command(["git", "add", dvc_file]) - self._run_command(["git", "commit", "-m", message]) - - def tag_version(self, tag_name: str) -> None: - """Add a git tag for this version""" - self._run_command(["git", "tag", tag_name]) - - def push_to_remote(self) -> None: - """Push data to DVC remote and metadata to git""" - self._run_command(["dvc", "push"]) - self._run_command(["git", "push", "--follow-tags"]) - - def get_version(self) -> str: - """Get the current version from DVC""" - result = self._run_command(["dvc", "version"]) - return str(result.stdout.strip()) - - def get_remote(self) -> str: - """Get the current remote from DVC""" - result = self._run_command(["dvc", "remote", "list"]) - return str(result.stdout.strip()) - - def setup_remote(self, remote_name: str, remote_url: str) -> None: - """Configure a DVC remote for storing data""" - self._run_command(["dvc", "remote", "add", remote_name, remote_url]) - self._run_command(["dvc", "remote", "default", remote_name]) - - def verify_file(self, file_path: Union[str, Path]) -> bool: - """Verify file integrity using DVC checksums""" - rel_path = Path(file_path).relative_to(self.repo_path) - try: - result = self._run_command(["dvc", "status", str(rel_path)]) - return "up to date" in str(result.stdout) - except subprocess.CalledProcessError: - return False - - def rollback_to_version( - self, file_path: Union[str, Path], version_tag: str - ) -> None: - """Roll back a file to a specific version using DVC and Git""" - rel_path = Path(file_path).relative_to(self.repo_path) - dvc_file = str(rel_path) + ".dvc" - - # Checkout the specific version of the DVC file - self._run_command(["git", "checkout", version_tag, "--", dvc_file]) - - # Pull the data for that version - self._run_command(["dvc", "checkout", str(rel_path)]) - - def lock_resource(self, file_path: Union[str, Path]) -> None: - """Lock a resource to prevent concurrent modifications""" - rel_path = Path(file_path).relative_to(self.repo_path) - self._run_command(["dvc", "lock", str(rel_path)]) - - def unlock_resource(self, file_path: Union[str, Path]) -> None: - """Unlock a previously locked resource""" - rel_path = Path(file_path).relative_to(self.repo_path) - self._run_command(["dvc", "unlock", str(rel_path)]) - - def add_metric(self, metric_file: Union[str, Path], metric_name: str) -> None: - """Add a file as a DVC metric for tracking""" - rel_path = Path(metric_file).relative_to(self.repo_path) - self._run_command( - ["dvc", "metrics", "add", str(rel_path), "--name", metric_name] - ) - - def show_metrics(self) -> str: - """Show all tracked metrics""" - result = self._run_command(["dvc", "metrics", "show"]) - return str(result.stdout.strip()) - - def gc_cache(self, force: bool = False) -> None: - """Clean up unused cache to save disk space""" - cmd = ["dvc", "gc"] - if force: - cmd.append("-f") - self._run_command(cmd) - - def configure(self, section: str, option: str, value: str) -> None: - """Configure DVC settings""" - self._run_command(["dvc", "config", f"{section}.{option}", value]) - - def has_changes(self, file_path: Union[str, Path]) -> bool: - """Check if a file has uncommitted changes""" - rel_path = Path(file_path).relative_to(self.repo_path) - try: - result = self._run_command(["dvc", "status", str(rel_path)]) - # If there are changes, the output will contain the file path - return str(rel_path) in str(result.stdout) - except subprocess.CalledProcessError: - # If the command fails, assume there are changes - return True diff --git a/api/models/Resource.py b/api/models/Resource.py index 2af8d0a9..c9857edd 100644 --- a/api/models/Resource.py +++ b/api/models/Resource.py @@ -1,23 +1,17 @@ import hashlib -import os -import random import uuid -from pathlib import Path from typing import TYPE_CHECKING, Any, Optional import structlog -from django.conf import settings from django.contrib.auth import get_user_model from django.db import models -from django.db.models.signals import post_save, pre_save +from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils.text import slugify logger = structlog.getLogger(__name__) -from api.managers.dvc_manager import DVCManager from api.utils.enums import DataType -from api.utils.version_detection import detect_version_change_type if TYPE_CHECKING: from api.models.Dataset import Dataset @@ -168,160 +162,6 @@ def hash_resource_file(sender, instance: ResourceFileDetails, **kwargs): instance.sha256 = compute_sha256(instance.file) -@receiver(post_save, sender=ResourceFileDetails) -def version_resource_with_dvc(sender, instance: ResourceFileDetails, created, **kwargs): - """Create a new version using DVC when resource is updated""" - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - - # Skip if this is just being created (first version) - if created: - # For first version, we just track it - try: - # Use chunked mode for large files (over 100MB) - file_size = ( - instance.file.size - if hasattr(instance.file, "size") - else os.path.getsize(instance.file.path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - dvc_file = dvc.track_resource(instance.file.path, chunked=use_chunked) - message = f"Add resource: {instance.resource.name} version {instance.resource.version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{instance.resource.name}-{instance.resource.version}") - - # Create first version record - ResourceVersion.objects.create( - resource=instance.resource, - version_number=instance.resource.version, - change_description=f"Initial version of {instance.resource.name}", - ) - except Exception as e: - logger.error(f"Failed to version resource: {str(e)}") - # Continue without versioning if it fails - # This allows the resource to be created even if DVC fails - pass - else: - # For updates, check if the file has actually changed before creating a new version - try: - # Skip versioning if the file hasn't changed - if dvc.verify_file(instance.file.path): - logger.info( - f"No changes detected for {instance.resource.name}, skipping version creation" - ) - return - - # Get the latest version - last_version: Optional[ResourceVersion] = instance.resource.versions.order_by( - "-created_at" - ).first() - - # Handle case when there are no versions yet - if last_version is None: - new_version = "v1.0.0" - else: - # Determine appropriate version increment type by analyzing changes - try: - # Get the previous version file path - # We need to create a temporary copy of the previous version - import shutil - import tempfile - - from django.core.files.storage import default_storage - - # Create a temporary directory for the previous version - with tempfile.TemporaryDirectory() as temp_dir: - # Get the previous version file path - file_name = instance.file.name - if not file_name: - raise ValueError("File name is missing") - prev_file_name = ( - f"prev_version_{instance.resource.id}.{file_name.split('.')[-1]}" - ) - prev_file_path = os.path.join(temp_dir, prev_file_name) - - # Use DVC to get the previous version - try: - # Try to checkout the previous version using DVC - rel_path = Path(instance.file.path).relative_to(settings.DVC_REPO_PATH) - tag_name = f"{instance.resource.name}-{last_version.version_number}" - - # Save current file to temp location - current_file_path = instance.file.path - temp_current = os.path.join(temp_dir, "current_file") - shutil.copy2(current_file_path, temp_current) - - # Checkout previous version - dvc.rollback_to_version(instance.file.path, tag_name) - - # Copy the previous version to our temp location - shutil.copy2(instance.file.path, prev_file_path) - - # Restore current file - shutil.copy2(temp_current, current_file_path) - - # Detect version change type - increment_type = detect_version_change_type( - prev_file_path, current_file_path - ) - logger.info( - f"Detected version change type: {increment_type} for {instance.resource.name}" - ) - except Exception as e: - logger.warning( - f"Could not analyze version changes: {str(e)}, defaulting to minor version" - ) - increment_type = "minor" - except Exception as e: - logger.warning( - f"Error in version detection: {str(e)}, defaulting to minor version" - ) - increment_type = "minor" - - # Increment version based on detected change type - new_version = _increment_version( - last_version.version_number, increment_type=increment_type - ) - - # Use chunked mode for large files (over 100MB) - file_size = ( - instance.file.size - if hasattr(instance.file, "size") - else os.path.getsize(instance.file.path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - # Update using DVC - dvc_file = dvc.track_resource(instance.file.path, chunked=use_chunked) - message = f"Update resource: {instance.resource.name} to version {new_version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{instance.resource.name}-{new_version}") - - # Create version record - ResourceVersion.objects.create( - resource=instance.resource, - version_number=new_version, - change_description=f"Updated version of {instance.resource.name}", - ) - - # Update resource version field - instance.resource.version = new_version - instance.resource.save(update_fields=["version"]) - - # Optional: Trigger garbage collection periodically - # This could be moved to a scheduled task instead - if random.random() < 0.05: # 5% chance to run GC on version update - try: - dvc.gc_cache() - except Exception as e: - logger.warning(f"Failed to run garbage collection: {str(e)}") - except Exception as e: - logger.error(f"Failed to update resource version: {str(e)}") - # Continue without versioning if it fails - pass - - def _increment_version(version: str, increment_type: str = "minor") -> str: """Semantic version incrementing logic diff --git a/api/schema/resource_schema.py b/api/schema/resource_schema.py index 85fbc965..bb3c6d2a 100644 --- a/api/schema/resource_schema.py +++ b/api/schema/resource_schema.py @@ -9,11 +9,11 @@ import strawberry import strawberry_django import structlog +from django.db import transaction from django.db.models import QuerySet from strawberry.file_uploads import Upload from strawberry.types import Info -from api.managers.dvc_manager import DVCManager from api.models import ( Dataset, Resource, @@ -29,6 +29,9 @@ from api.utils.data_indexing import index_resource_data from api.utils.file_utils import file_validation from api.utils.graphql_telemetry import trace_resolver +from api.utils.graphql_utils import is_superuser +from authorization.permissions import IsAuthenticated +from authorization.services import AuthorizationService logger = structlog.get_logger("dataspace.resource_schema") @@ -393,6 +396,7 @@ def delete_file_resource(self, info: Info, resource_id: uuid.UUID) -> bool: @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[IsAuthenticated], extensions=[ TrackModelActivity( verb="versioned", @@ -412,58 +416,37 @@ def create_major_version(self, info: Info, input: CreateMajorVersionInput) -> Ty This should be used when significant changes are made to the resource data structure, such as schema changes, column additions/removals, or other breaking changes. """ - import os - - from django.conf import settings - try: - # Get the resource - resource = Resource.objects.get(id=input.resource_id) + resource = Resource.objects.select_related("dataset").get(id=input.resource_id) except Resource.DoesNotExist: raise ValueError(f"Resource with ID {input.resource_id} does not exist") - # Get the latest version - last_version = resource.versions.order_by("-created_at").first() + # Only the dataset's owner or someone who may change it can version its resources. + user = info.context.user + dataset = resource.dataset + can_change = dataset.user_id == user.id or AuthorizationService.check_dataset_permission( + user.id, str(dataset.id), "change" + ) + if not is_superuser(info) and not can_change: + raise PermissionError("You don't have permission to version this resource.") + last_version = resource.versions.order_by("-created_at").first() if not last_version: logger.warning( f"No previous version found for resource {resource.name}, creating initial version" ) new_version = "v1.0.0" else: - # Increment major version new_version = _increment_version(last_version.version_number, increment_type="major") - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - - # Get the resource file path - file_path = resource.resourcefiledetails.file.path - - # Determine if file is large and should use chunking - file_size = ( - resource.resourcefiledetails.file.size - if hasattr(resource.resourcefiledetails.file, "size") - else os.path.getsize(file_path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - # Track with DVC - dvc_file = dvc.track_resource(file_path, chunked=use_chunked) - message = f"Major version update for resource: {resource.name} to version {new_version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{resource.name}-{new_version}") - - # Create version record - ResourceVersion.objects.create( - resource=resource, - version_number=new_version, - change_description=input.description, - ) - - # Update resource version field - resource.version = new_version - resource.save(update_fields=["version"]) + with transaction.atomic(): + ResourceVersion.objects.create( + resource=resource, + version_number=new_version, + change_description=input.description, + ) + resource.version = new_version + resource.save(update_fields=["version"]) logger.info(f"Created major version {new_version} for resource {resource.name}") return TypeResource.from_django(resource) diff --git a/api/signals/dataset_signals.py b/api/signals/dataset_signals.py index 3d4ac987..6f9f88ee 100644 --- a/api/signals/dataset_signals.py +++ b/api/signals/dataset_signals.py @@ -1,11 +1,9 @@ from typing import Any, Optional import structlog -from django.conf import settings from django.db.models.signals import pre_save from django.dispatch import receiver -from api.managers.dvc_manager import DVCManager from api.models.Dataset import Dataset from api.models.Resource import Resource, ResourceVersion, _increment_version from api.utils.enums import DatasetStatus @@ -77,16 +75,13 @@ def handle_dataset_publication(sender: Any, instance: Dataset, **kwargs: Any) -> # Get all resources for this dataset resources = instance.resources.all() - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - # Increment version for each resource for resource in resources: # type: Resource try: # Get the latest version - last_version: Optional[ResourceVersion] = ( - resource.versions.order_by("-created_at").first() - ) + last_version: Optional[ResourceVersion] = resource.versions.order_by( + "-created_at" + ).first() if last_version: # Increment minor version (publication is a significant change) @@ -94,17 +89,6 @@ def handle_dataset_publication(sender: Any, instance: Dataset, **kwargs: Any) -> last_version.version_number, increment_type="minor" ) - # Get the resource file path - file_path: str = resource.resourcefiledetails.file.path - - # Track with DVC - dvc_file = dvc.track_resource(file_path) - message: str = ( - f"Publishing resource: {resource.name} to version {new_version}" - ) - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{resource.name}-{new_version}") - # Create version record ResourceVersion.objects.create( resource=resource, diff --git a/api/utils/version_detection.py b/api/utils/version_detection.py deleted file mode 100644 index 809e9db6..00000000 --- a/api/utils/version_detection.py +++ /dev/null @@ -1,327 +0,0 @@ -import json -import os -import xml.etree.ElementTree as ET -from typing import Any, Dict, List, Set, Tuple, Union - -import pandas as pd -import structlog -from deepdiff import DeepDiff # type: ignore - -logger = structlog.getLogger(__name__) - - -def detect_version_change_type(old_file_path: str, new_file_path: str) -> str: - """ - Analyze changes between two versions of a file and determine the appropriate - version increment type (major, minor, or patch). - - Args: - old_file_path: Path to the previous version of the file - new_file_path: Path to the new version of the file - - Returns: - String indicating the type of version change: "major", "minor", or "patch" - """ - # Get file extensions - _, old_ext = os.path.splitext(old_file_path) - _, new_ext = os.path.splitext(new_file_path) - - # If file format changed, it's a major change - if old_ext.lower() != new_ext.lower(): - logger.info( - f"File format changed from {old_ext} to {new_ext} - major version change" - ) - return "major" - - # Handle different file types - if old_ext.lower() in [".csv", ".tsv"]: - return _detect_tabular_changes(old_file_path, new_file_path) - elif old_ext.lower() in [".json"]: - return _detect_json_changes(old_file_path, new_file_path) - elif old_ext.lower() in [".xml"]: - return _detect_xml_changes(old_file_path, new_file_path) - else: - # For other file types, default to comparing file size - return _detect_generic_changes(old_file_path, new_file_path) - - -def _detect_tabular_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in tabular data files (CSV, TSV) using pandas - """ - try: - # Read files with pandas - old_df = pd.read_csv(old_file_path, encoding="utf-8") - new_df = pd.read_csv(new_file_path, encoding="utf-8") - - # Check for schema changes - old_columns = set(old_df.columns) - new_columns = set(new_df.columns) - - # If columns were added or removed, it's a major change - if old_columns != new_columns: - added = new_columns - old_columns - removed = old_columns - new_columns - logger.info( - f"Schema changed - added columns: {added}, removed columns: {removed} - major version change" - ) - return "major" - - # Check for data type changes in columns - for col in old_columns: - if old_df[col].dtype != new_df[col].dtype: - logger.info( - f"Data type changed for column {col} - major version change" - ) - return "major" - - # Check for significant row count changes - if len(old_df) != len(new_df): - # Row count changed by more than 10% - change_percentage = ( - abs(len(new_df) - len(old_df)) / max(len(old_df), 1) * 100 - ) - if change_percentage > 10: - logger.info( - f"Row count changed by {change_percentage:.2f}% - minor version change" - ) - return "minor" - - # Use pandas built-in comparison for data changes - # Limit to common rows to avoid index errors - min_rows = min(len(old_df), len(new_df)) - if min_rows > 0: - # Truncate both dataframes to the same length - old_sample = old_df.iloc[:min_rows].reset_index(drop=True) - new_sample = new_df.iloc[:min_rows].reset_index(drop=True) - - # Use pandas compare function (available in pandas >= 1.1.0) - try: - # Get differences between dataframes - diff_df = new_sample.compare(old_sample) - - # Calculate change percentage - total_cells = min_rows * len(old_columns) - changed_cells = diff_df.size # Number of changed cells - - change_percentage = ( - (changed_cells / total_cells) * 100 if total_cells > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"Data changed significantly ({change_percentage:.2f}% of cells) - minor version change" - ) - return "minor" - elif change_percentage > 0: - logger.info( - f"Small data changes ({change_percentage:.2f}% of cells) - patch version change" - ) - return "patch" - except Exception as compare_error: - # Fallback if compare fails (older pandas version or other issues) - logger.warning( - f"Pandas compare failed: {str(compare_error)}, using manual comparison" - ) - - # Manual comparison as fallback - changes = 0 - total_cells = min_rows * len(old_columns) - - for col in old_columns: - # Compare column values - unequal_mask = old_sample[col].values != new_sample[col].values - changes += unequal_mask.sum() # type: ignore - - change_percentage = ( - (changes / total_cells) * 100 if total_cells > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"Data changed significantly ({change_percentage:.2f}% of cells) - minor version change" - ) - return "minor" - elif change_percentage > 0: - logger.info( - f"Small data changes ({change_percentage:.2f}% of cells) - patch version change" - ) - return "patch" - - # Default to patch for minimal changes - return "patch" - - except Exception as e: - logger.error(f"Error analyzing tabular files: {str(e)}") - # Default to minor if we can't analyze properly - return "minor" - - -def _detect_json_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in JSON files using DeepDiff - """ - try: - with open(old_file_path, "r", encoding="utf-8") as f: - old_data = json.load(f) - - with open(new_file_path, "r", encoding="utf-8") as f: - new_data = json.load(f) - - # Use DeepDiff for efficient and accurate comparison - diff = DeepDiff(old_data, new_data, ignore_order=True) - - # Check for structure changes (major) - if diff.get("dictionary_item_added") or diff.get("dictionary_item_removed"): - logger.info(f"JSON structure changed - major version change") - return "major" - - # Check for type changes (major) - if diff.get("type_changes"): - logger.info(f"JSON data types changed - major version change") - return "major" - - # Check for significant value changes (minor) - value_changes = len(diff.get("values_changed", {})) - if isinstance(old_data, dict) and isinstance(new_data, dict): - total_values = _count_values(old_data) - change_percentage = ( - (value_changes / total_values) * 100 if total_values > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"JSON values changed significantly ({change_percentage:.2f}%) - minor version change" - ) - return "minor" - - # Check for array changes (minor) - if diff.get("iterable_item_added") or diff.get("iterable_item_removed"): - logger.info(f"JSON array items changed - minor version change") - return "minor" - - # If there are any changes at all, it's at least a patch - if diff: - return "patch" - - # No changes detected - return "patch" - - except Exception as e: - logger.error(f"Error analyzing JSON files: {str(e)}") - return "minor" - - -def _detect_xml_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in XML files - """ - try: - import xml.etree.ElementTree as ET - - old_tree = ET.parse(old_file_path) - new_tree = ET.parse(new_file_path) - - old_root = old_tree.getroot() - new_root = new_tree.getroot() - - # Check for tag changes - old_tags = _get_xml_tags(old_root) - new_tags = _get_xml_tags(new_root) - - if old_tags != new_tags: - logger.info(f"XML structure changed - major version change") - return "major" - - # Check for attribute changes - old_attrs = _get_xml_attributes(old_root) - new_attrs = _get_xml_attributes(new_root) - - if old_attrs != new_attrs: - logger.info(f"XML attributes changed - minor version change") - return "minor" - - # Default to patch for content changes - return "patch" - - except Exception as e: - logger.error(f"Error analyzing XML files: {str(e)}") - return "minor" - - -def _detect_generic_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in generic files based on size - """ - old_size = os.path.getsize(old_file_path) - new_size = os.path.getsize(new_file_path) - - # Calculate size change percentage - size_change_percentage = abs(new_size - old_size) / max(old_size, 1) * 100 - - if size_change_percentage > 50: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - major version change" - ) - return "major" - elif size_change_percentage > 10: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - minor version change" - ) - return "minor" - else: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - patch version change" - ) - return "patch" - - -def _flatten_dict_keys(d: Dict, parent_key: str = "") -> List[str]: - """ - Flatten nested dictionary keys - """ - items = [] - for k, v in d.items(): - new_key = f"{parent_key}.{k}" if parent_key else k - if isinstance(v, dict): - items.extend(_flatten_dict_keys(v, new_key)) - else: - items.append(new_key) - return items - - -def _count_values(obj: Union[Dict, List, Any], count: int = 0) -> int: - """ - Count the total number of values in a nested structure - """ - if isinstance(obj, dict): - for k, v in obj.items(): - count += 1 # Count the key-value pair itself - if isinstance(v, (dict, list)): - count = _count_values(v, count) - elif isinstance(obj, list): - count += len(obj) # Count the list items - for item in obj: - if isinstance(item, (dict, list)): - count = _count_values(item, count) - return count - - -def _get_xml_tags(element: ET.Element) -> Set[str]: - """ - Get all unique tag names in an XML tree - """ - tags = {element.tag} - for child in element: - tags.update(_get_xml_tags(child)) - return tags - - -def _get_xml_attributes(element: ET.Element) -> Set[Tuple[str, str]]: - """ - Get all attributes in an XML tree as (element_tag, attribute_name) pairs - """ - attrs = {(element.tag, attr) for attr in element.attrib.keys()} - for child in element: - attrs.update(_get_xml_attributes(child)) - return attrs diff --git a/docs/version_detection_system.md b/docs/version_detection_system.md deleted file mode 100644 index 66fbd99f..00000000 --- a/docs/version_detection_system.md +++ /dev/null @@ -1,140 +0,0 @@ -# Intelligent Version Detection System - -## Overview - -The DataSpace platform implements an version detection system that automatically determines the appropriate version increment (major, minor, or patch) when resources are updated. This document explains the technical implementation, triggering mechanisms, and version classification logic. - -## Version Increment Types - -The system follows semantic versioning principles (X.Y.Z): - -1. **Major Version (X)**: Breaking changes or significant structural modifications -2. **Minor Version (Y)**: Publishing Dataset or significant data changes that maintain compatibility -3. **Patch Version (Z)**: Small fixes, corrections, or minimal data changes - -## Triggering Mechanisms - -Version changes are automatically triggered by the following events: - -1. **Resource File Updates**: When a resource file is updated through the API -2. **Dataset Publication**: When a dataset containing resources is published -3. **Manual Triggers**: Through management commands (e.g., `create_major_version`) - -## Technical Implementation - -### Components - -1. **Signal Handlers**: Detect changes to resources and trigger version detection -2. **Version Detection Utility**: Analyzes file changes to determine version increment type -3. **DVC Manager**: Handles version tracking, tagging, and remote storage - -### Signal Flow - -``` -ResourceFileDetails update → post_save signal → version_resource_with_dvc - → detect_version_change_type → _increment_version → Create ResourceVersion -``` - -## Change Detection Logic - -The system uses different strategies based on file type: - -### CSV/Tabular Files - -**Major Version** triggers: -- Schema changes (columns added/removed) -- Data type changes in columns - -**Minor Version** triggers: -- Row count changes > 10% -- Data changes in > 30% of cells - -**Patch Version** triggers: -- Small data changes (< 30% of cells) -- Minimal corrections - -### JSON Files - -**Major Version** triggers: -- Structure changes (keys added/removed) -- Data type changes - -**Minor Version** triggers: -- Significant value changes (> 30% of values) -- Array item additions/removals - -**Patch Version** triggers: -- Small value changes -- Formatting changes - -### XML Files - -**Major Version** triggers: -- Tag structure changes - -**Minor Version** triggers: -- Attribute changes - -**Patch Version** triggers: -- Content changes without structural modifications - -### Generic Files - -For non-structured files, the system uses file size differences: - -**Major Version** triggers: -- Size changes > 50% - -**Minor Version** triggers: -- Size changes between 10% and 50% - -**Patch Version** triggers: -- Size changes < 10% - -## Technical Details - -### Dependencies - -- **pandas**: For efficient tabular data comparison -- **DeepDiff**: For accurate JSON structure comparison -- **DVC**: For version tracking and storage - -### Performance Considerations - -- Large files (>100MB) use chunked processing -- Tabular comparisons use sampling for very large datasets -- Early return logic to avoid unnecessary processing - -## Example - -When a CSV resource is updated: - -1. The system detects the change through a Django signal -2. `detect_version_change_type` loads both versions of the file -3. The function compares schemas, data types, and values -4. Based on the extent of changes, it returns "major", "minor", or "patch" -5. The version is incremented accordingly (e.g., 1.2.3 → 1.3.0 for minor) -6. DVC tracks the new version with appropriate tags -7. A ResourceVersion record is created in the database - -## Management Commands - -The system includes management commands for manual version control: - -- `create_major_version`: Force a major version increment for a resource -- `setup_dvc`: Configure DVC repository and remotes - -## Error Handling - -The version detection system includes robust error handling to ensure that: - -1. Failed comparisons default to "minor" version changes -2. Temporary files are properly cleaned up -3. Errors are logged with detailed context -4. The system continues functioning even if analysis fails - -## Future Improvements - -- Support for more file formats -- Configurable thresholds for different file types -- Performance optimizations for very large datasets diff --git a/requirements.txt b/requirements.txt index ca9b8bb5..3b5da7ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,6 @@ django-elasticsearch-dsl==8.0 django-elasticsearch-dsl-drf==0.22.5 django-nine==0.2.7 djangorestframework==3.14.0 -dvc==3.59.1 elastic-transport==8.13.0 elasticsearch==8.13.0 elasticsearch-dsl==8.12.0 From 0473100418cda7632d6eedcefff2aa0b0c30953e Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:10:44 +0530 Subject: [PATCH 08/55] feat(storage): add S3 public/private media storages Media moves to S3 only, through django-storages. Two storages share one bucket under /public and /private: public assets get unsigned, cacheable URLs; private files get 5-minute presigned URLs. The default storage is private, so anything that falls back to it is never world-readable by accident. Empty access keys use the boto3 credential chain (ECS/EC2 task role). boto/urllib3 loggers are capped at WARNING because at DEBUG they log every signed request, Authorization header included. Tests swap all storages for InMemoryStorage and never reach S3. --- .env.example | 17 ++++++----- DataSpace/settings.py | 48 ++++++++++++++++++++++++++++++-- api/utils/storage.py | 35 +++++++++++++++++++++++ requirements.txt | 3 ++ tests/test_publication_blocks.py | 18 ++++-------- tests/test_settings.py | 14 ++++++++++ 6 files changed, 113 insertions(+), 22 deletions(-) create mode 100644 api/utils/storage.py diff --git a/.env.example b/.env.example index 136af89e..3de8e087 100644 --- a/.env.example +++ b/.env.example @@ -23,13 +23,16 @@ GITHUB_TOKEN= PUBLIC_SITE_URL=https://civicdataspace.in PUBLIC_API_URL=https://api.datakeep.civicdays.in -# --- s3 only --- -# STORAGE_BUCKET=dataspace-dev -# STORAGE_REGION=ap-south-1 -# STORAGE_ENV=dev -# STORAGE_PUBLIC_BASE_URL= -# STORAGE_ACCESS_KEY_ID= -# STORAGE_SECRET_ACCESS_KEY= +# Media storage (S3 only; required). STORAGE_ENV prefixes every key: use +# "local" for local development against the dev bucket. Leave the access keys +# empty on ECS/EC2 so the task role is used. STORAGE_PUBLIC_BASE_URL is the +# base for public asset URLs (CDN or bucket URL); empty uses the S3 bucket URL. +STORAGE_BUCKET=dataspace-dev +STORAGE_REGION=ap-south-1 +STORAGE_ENV=local +STORAGE_PUBLIC_BASE_URL= +STORAGE_ACCESS_KEY_ID= +STORAGE_SECRET_ACCESS_KEY= # Per-IP request throttling (api/middleware/rate_limit.py). False on dev only. RATE_LIMIT_ENABLED=True diff --git a/DataSpace/settings.py b/DataSpace/settings.py index 63452733..9922233d 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -255,8 +255,39 @@ # https://docs.djangoproject.com/en/4.0/howto/static-files/ # This STATIC_URL setting is overridden below +# Uploaded media lives only in S3 (see STORAGES below). MEDIA_URL is kept as the +# prefix of the legacy /public/ redirect route for links issued before S3. MEDIA_URL = "public/" -MEDIA_ROOT = os.path.join(BASE_DIR, "files", "public") + +# S3 media storage. STORAGE_ENV prefixes every key (local, dev, prod) so one +# bucket can hold several environments. Empty keys fall back to the boto3 +# credential chain, which is what ECS/EC2 task roles use. +STORAGE_BUCKET = env("STORAGE_BUCKET") +STORAGE_REGION = env("STORAGE_REGION") +STORAGE_ENV = env("STORAGE_ENV") +STORAGE_PUBLIC_BASE_URL = env("STORAGE_PUBLIC_BASE_URL", default="") + +_S3_OPTIONS = { + "bucket_name": STORAGE_BUCKET, + "region_name": STORAGE_REGION, + "access_key": env("STORAGE_ACCESS_KEY_ID", default="") or None, + "secret_key": env("STORAGE_SECRET_ACCESS_KEY", default="") or None, + "file_overwrite": False, + # The bucket enforces object ownership; access is granted by bucket policy, not ACLs. + "default_acl": None, +} +_S3_PUBLIC_OPTIONS = { + **_S3_OPTIONS, + "location": f"{STORAGE_ENV}/public", + "querystring_auth": False, + "custom_domain": STORAGE_PUBLIC_BASE_URL.split("://", 1)[-1].rstrip("/") or None, +} +_S3_PRIVATE_OPTIONS = { + **_S3_OPTIONS, + "location": f"{STORAGE_ENV}/private", + "querystring_auth": True, + "querystring_expire": 300, +} # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field @@ -372,6 +403,9 @@ "level": "DEBUG" if DEBUG else "INFO", "propagate": True, }, + # At DEBUG the AWS SDK logs every signed request, Authorization header + # included, and floods the log on each S3 call. + **{name: {"level": "WARNING"} for name in ("boto3", "botocore", "s3transfer", "urllib3")}, }, } @@ -485,8 +519,16 @@ "whitenoise.middleware.WhiteNoiseMiddleware", ) -# Use the simplest WhiteNoise storage configuration to avoid MIME type issues -STATICFILES_STORAGE = "whitenoise.storage.StaticFilesStorage" +# "public"/"private" are what model fields name through api.utils.storage. The +# default is private so anything that falls back to default_storage is never +# world-readable by accident. +STORAGES = { + "default": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS}, + "public": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PUBLIC_OPTIONS}, + "private": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS}, + # Use the simplest WhiteNoise storage configuration to avoid MIME type issues + "staticfiles": {"BACKEND": "whitenoise.storage.StaticFilesStorage"}, +} # Disable compression to avoid MIME type issues WHITENOISE_ENABLE_COMPRESSION = False diff --git a/api/utils/storage.py b/api/utils/storage.py new file mode 100644 index 00000000..7cdb679e --- /dev/null +++ b/api/utils/storage.py @@ -0,0 +1,35 @@ +"""Media storages for model file fields. + +Every ``FileField``/``ImageField`` names one of these through ``storage=``. +They are callables, not instances, so migrations record a function reference +instead of bucket settings, and tests can swap the backend through STORAGES. +""" + +from django.core.files.storage import Storage, storages + +# Folder prefixes from the on-disk era, when MEDIA_ROOT was files/public and the +# upload_to helpers added their own "files/..." on top. The S3 storages already +# put every key under /public or /private, so these only repeat it. +_LEGACY_PREFIXES = ("files/public/", "files/") + + +def public_storage() -> Storage: + """Storage for files anyone may read: unsigned, cacheable URLs.""" + return storages["public"] + + +def private_storage() -> Storage: + """Storage for gated files: short-lived presigned URLs only.""" + return storages["private"] + + +def normalize_media_name(name: str) -> str: + """Strip the legacy ``files/public/`` or ``files/`` prefix from a stored name. + + ``files/public/organizations/acme/png/logo.png`` -> ``organizations/acme/png/logo.png``. + Names without a legacy prefix are returned unchanged. + """ + for prefix in _LEGACY_PREFIXES: + if name.startswith(prefix): + return name[len(prefix) :] + return name diff --git a/requirements.txt b/requirements.txt index 3b5da7ea..cb69c412 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,9 @@ django-elasticsearch-dsl==8.0 django-elasticsearch-dsl-drf==0.22.5 django-nine==0.2.7 djangorestframework==3.14.0 +# S3 media storage +django-storages[s3]==1.14.4 +boto3==1.43.108 elastic-transport==8.13.0 elasticsearch==8.13.0 elasticsearch-dsl==8.12.0 diff --git a/tests/test_publication_blocks.py b/tests/test_publication_blocks.py index 6364830e..38ee1b98 100644 --- a/tests/test_publication_blocks.py +++ b/tests/test_publication_blocks.py @@ -1,6 +1,5 @@ """Layer 1/3/4 tests for content blocks: add/reorder/remove/replace + download gate.""" -import os from datetime import date import pytest @@ -24,11 +23,6 @@ VIDEO = "https://youtu.be/dQw4w9WgXcQ" -@pytest.fixture(autouse=True) -def media_root(settings, tmp_path): - settings.MEDIA_ROOT = str(tmp_path) - - @pytest.fixture def owner(db): return User.objects.create(username="owner", keycloak_id="owner") @@ -117,27 +111,27 @@ def test_reorder_sets_positions_to_requested_order(self, publication): class TestReplaceFile: def test_replace_swaps_file_and_removes_the_old_one(self, publication): block = add_file_block(publication, _pdf("first.pdf")) - old_path = block.file.path + storage, old_name = block.file.storage, block.file.name old_id = block.id - assert os.path.exists(old_path) + assert storage.exists(old_name) replace_block_file(block, _pdf("second.pdf")) assert block.id == old_id # same row, in-place assert block.file_name == "second.pdf" - assert not os.path.exists(old_path) # old file removed from disk + assert not storage.exists(old_name) # old file removed from storage @pytest.mark.django_db class TestDeleteRemovesFile: def test_deleting_a_block_removes_its_file(self, publication): block = add_file_block(publication, _pdf()) - path = block.file.path - assert os.path.exists(path) + storage, name = block.file.storage, block.file.name + assert storage.exists(name) block.delete() - assert not os.path.exists(path) # post_delete signal cleaned it up + assert not storage.exists(name) # post_delete signal cleaned it up # --------------------------------------------------------------------------- # diff --git a/tests/test_settings.py b/tests/test_settings.py index 6a641855..05375f6d 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -9,8 +9,22 @@ project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, project_root) +# The real settings require the S3 storage variables; give the suite harmless +# values so it boots without a .env. STORAGES is swapped for memory below. +os.environ.setdefault("STORAGE_BUCKET", "test-bucket") +os.environ.setdefault("STORAGE_REGION", "ap-south-1") +os.environ.setdefault("STORAGE_ENV", "test") + from DataSpace.settings import * +# Tests never reach S3: both media storages (and the default) live in memory. +STORAGES = { + **STORAGES, # noqa: F405 — imported via ``from DataSpace.settings import *`` + "default": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, + "public": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, + "private": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, +} + # Use an in-memory SQLite database for testing DATABASES = { "default": { From 73f538135c536a6b8532a2cdb61e81a6cc4036be Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:10:51 +0530 Subject: [PATCH 09/55] feat(storage): store model file fields in S3 Every FileField/ImageField now names a storage: logos, cover images, profile pictures and publication block files go to public storage; resource files and chart images go to private storage. The storages are callables, so the migrations record a function reference, not bucket settings. upload_to paths drop their "files/public/" and "files/" prefixes, which only repeated what MEDIA_ROOT used to add; the storages already put every key under /public or /private. --- api/migrations/0054_file_field_storages.py | 108 ++++++++++++++++++ api/models/Catalog.py | 5 +- api/models/Collaborative.py | 13 ++- api/models/DataSpace.py | 5 +- api/models/Organization.py | 11 +- api/models/Publication.py | 2 + api/models/Resource.py | 3 +- api/models/ResourceChartImage.py | 7 +- api/models/UseCase.py | 11 +- api/utils/file_paths.py | 19 +-- .../0005_profile_picture_storage.py | 26 +++++ authorization/models.py | 6 +- 12 files changed, 193 insertions(+), 23 deletions(-) create mode 100644 api/migrations/0054_file_field_storages.py create mode 100644 authorization/migrations/0005_profile_picture_storage.py diff --git a/api/migrations/0054_file_field_storages.py b/api/migrations/0054_file_field_storages.py new file mode 100644 index 00000000..4e8c6f2b --- /dev/null +++ b/api/migrations/0054_file_field_storages.py @@ -0,0 +1,108 @@ +# Generated by Django 5.0.4 on 2026-10-12 20:25 + +from django.db import migrations, models + +import api.utils.file_paths +import api.utils.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.AlterField( + model_name="catalog", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._catalog_directory_path, + ), + ), + migrations.AlterField( + model_name="collaborative", + name="cover_image", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + migrations.AlterField( + model_name="collaborative", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + migrations.AlterField( + model_name="dataspace", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._dataspace_directory_path, + ), + ), + migrations.AlterField( + model_name="organization", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._organization_directory_path, + ), + ), + migrations.AlterField( + model_name="publicationblock", + name="file", + field=models.FileField( + blank=True, + max_length=300, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._publication_block_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcechartimage", + name="image", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.private_storage, + upload_to=api.utils.file_paths._chart_image_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcefiledetails", + name="file", + field=models.FileField( + max_length=300, storage=api.utils.storage.private_storage, upload_to="resources/" + ), + ), + migrations.AlterField( + model_name="usecase", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + ] diff --git a/api/models/Catalog.py b/api/models/Catalog.py index 101bb3d7..799a82f6 100644 --- a/api/models/Catalog.py +++ b/api/models/Catalog.py @@ -6,12 +6,15 @@ from django.utils.translation import gettext_lazy as _ from api.utils.file_paths import _catalog_directory_path +from api.utils.storage import public_storage class Catalog(models.Model): name = models.CharField(max_length=255) datasets = models.ManyToManyField("api.Dataset", related_name="catalogs") - logo = models.ImageField(upload_to=_catalog_directory_path, blank=True, null=True) + logo = models.ImageField( + upload_to=_catalog_directory_path, storage=public_storage, blank=True, null=True + ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 3b9a38a7..f39f04a2 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -13,6 +13,7 @@ from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path +from api.utils.storage import public_storage slug_validator = RegexValidator( regex=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", @@ -25,10 +26,18 @@ class Collaborative(models.Model): title = models.CharField(max_length=200, unique=True, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) cover_image = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) diff --git a/api/models/DataSpace.py b/api/models/DataSpace.py index 7430065f..1cb9a8fc 100644 --- a/api/models/DataSpace.py +++ b/api/models/DataSpace.py @@ -4,12 +4,15 @@ from django.utils.text import slugify from api.utils.file_paths import _dataspace_directory_path +from api.utils.storage import public_storage class DataSpace(models.Model): name = models.CharField(max_length=200) description = models.CharField(max_length=1000) - logo = models.ImageField(upload_to=_dataspace_directory_path, blank=True, null=True) + logo = models.ImageField( + upload_to=_dataspace_directory_path, storage=public_storage, blank=True, null=True + ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) diff --git a/api/models/Organization.py b/api/models/Organization.py index 977f041a..96131f2d 100644 --- a/api/models/Organization.py +++ b/api/models/Organization.py @@ -5,21 +5,24 @@ from api.utils.enums import OrganizationTypes from api.utils.file_paths import _organization_directory_path +from api.utils.storage import public_storage class Organization(models.Model): name = models.CharField(max_length=200) description = models.CharField(max_length=1000) logo = models.ImageField( - upload_to=_organization_directory_path, blank=True, null=True, max_length=300 + upload_to=_organization_directory_path, + storage=public_storage, + blank=True, + null=True, + max_length=300, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - organization_types = models.CharField( - max_length=100, choices=OrganizationTypes.choices - ) + organization_types = models.CharField(max_length=100, choices=OrganizationTypes.choices) parent = models.ForeignKey( "api.Organization", unique=False, diff --git a/api/models/Publication.py b/api/models/Publication.py index 2acd2c5d..aea63f46 100644 --- a/api/models/Publication.py +++ b/api/models/Publication.py @@ -7,6 +7,7 @@ from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus from api.utils.file_paths import _publication_block_directory_path +from api.utils.storage import public_storage if TYPE_CHECKING: from api.models.Organization import Organization @@ -149,6 +150,7 @@ class PublicationBlock(models.Model): # FILE block fields. file = models.FileField( upload_to=_publication_block_directory_path, + storage=public_storage, max_length=300, blank=True, ) diff --git a/api/models/Resource.py b/api/models/Resource.py index c9857edd..9d20d934 100644 --- a/api/models/Resource.py +++ b/api/models/Resource.py @@ -12,6 +12,7 @@ logger = structlog.getLogger(__name__) from api.utils.enums import DataType +from api.utils.storage import private_storage if TYPE_CHECKING: from api.models.Dataset import Dataset @@ -67,7 +68,7 @@ def __str__(self) -> str: class ResourceFileDetails(models.Model): resource = models.OneToOneField(Resource, on_delete=models.CASCADE, null=False, blank=False) - file = models.FileField(upload_to="resources/", max_length=300) + file = models.FileField(upload_to="resources/", storage=private_storage, max_length=300) size = models.FloatField(blank=True, null=True) sha256 = models.CharField( max_length=64, diff --git a/api/models/ResourceChartImage.py b/api/models/ResourceChartImage.py index 16812dbb..9c19256d 100644 --- a/api/models/ResourceChartImage.py +++ b/api/models/ResourceChartImage.py @@ -6,6 +6,7 @@ # Import the Dataset model indirectly to avoid duplicate import error from api.utils.file_paths import _chart_image_directory_path +from api.utils.storage import private_storage class ResourceChartImage(models.Model): @@ -13,7 +14,11 @@ class ResourceChartImage(models.Model): name = models.CharField(max_length=50, unique=False, blank=True) description = models.CharField(max_length=1000, unique=False, blank=True) image = models.ImageField( - upload_to=_chart_image_directory_path, blank=True, null=True, max_length=300 + upload_to=_chart_image_directory_path, + storage=private_storage, + blank=True, + null=True, + max_length=300, ) dataset = models.ForeignKey( "api.Dataset", diff --git a/api/models/UseCase.py b/api/models/UseCase.py index 321bfdfc..0ea9b67d 100644 --- a/api/models/UseCase.py +++ b/api/models/UseCase.py @@ -16,6 +16,7 @@ UseCaseStatus, ) from api.utils.file_paths import _use_case_directory_path +from api.utils.storage import public_storage class UseCase(models.Model): @@ -23,7 +24,11 @@ class UseCase(models.Model): title = models.CharField(max_length=200, unique=True, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -71,9 +76,7 @@ def disambiguate(attempt: int) -> None: self.title = f"{base_title} ({attempt})" self.slug = slugify(cast(str, self.title)) - retry_on_slug_collision( - lambda: super(UseCase, self).save(*args, **kwargs), disambiguate - ) + retry_on_slug_collision(lambda: super(UseCase, self).save(*args, **kwargs), disambiguate) @property def is_individual_usecase(self): diff --git a/api/utils/file_paths.py b/api/utils/file_paths.py index aa1ecaa2..7cddf28b 100644 --- a/api/utils/file_paths.py +++ b/api/utils/file_paths.py @@ -2,6 +2,9 @@ import uuid from typing import Any +# Names are relative to the storage, which already prefixes every key with +# /public or /private; see api/utils/storage.py. + def _organization_directory_path(org: Any, filename: str) -> str: """ @@ -11,7 +14,7 @@ def _organization_directory_path(org: Any, filename: str) -> str: org_name = org.name _, extension = os.path.splitext(filename) - return f"files/public/organizations/{org_name}/{extension[1:]}/{filename}" + return f"organizations/{org_name}/{extension[1:]}/{filename}" def _chart_image_directory_path(chart: Any, filename: str) -> str: @@ -22,7 +25,7 @@ def _chart_image_directory_path(chart: Any, filename: str) -> str: chart_name = chart.name _, extension = os.path.splitext(filename) - return f"files/chart_images/{chart_name}/{extension[1:]}/{filename}" + return f"chart_images/{chart_name}/{extension[1:]}/{filename}" def _dataspace_directory_path(ds: Any, filename: str) -> str: @@ -33,7 +36,7 @@ def _dataspace_directory_path(ds: Any, filename: str) -> str: ds_name = ds.name _, extension = os.path.splitext(filename) - return f"files/public/dataspace/{ds_name}/{extension[1:]}/{filename}" + return f"dataspace/{ds_name}/{extension[1:]}/{filename}" def _use_case_directory_path(uc: Any, filename: str) -> str: @@ -44,7 +47,7 @@ def _use_case_directory_path(uc: Any, filename: str) -> str: uc_name = uc.title _, extension = os.path.splitext(filename) - return f"files/use_case/{uc_name}/logo/{filename}" + return f"use_case/{uc_name}/logo/{filename}" def _user_profile_image_directory_path(user: Any, filename: str) -> str: @@ -54,7 +57,7 @@ def _user_profile_image_directory_path(user: Any, filename: str) -> str: """ user_name = user.keycloak_id _, extension = os.path.splitext(filename) - return f"files/user_profile/{user_name}/{extension[1:]}/{filename}" + return f"user_profile/{user_name}/{extension[1:]}/{filename}" def _organization_file_directory_path(org: Any, filename: str) -> str: @@ -65,7 +68,7 @@ def _organization_file_directory_path(org: Any, filename: str) -> str: org_name = org.name _, extension = os.path.splitext(filename) - return f"files/resources/{org_name}/sample_data/{extension[1:]}/{filename}" + return f"resources/{org_name}/sample_data/{extension[1:]}/{filename}" def _catalog_directory_path(catalog: Any, filename: str) -> str: @@ -74,7 +77,7 @@ def _catalog_directory_path(catalog: Any, filename: str) -> str: """ catalog_name = catalog.name _, extension = os.path.splitext(filename) - return f"files/catalog/{catalog_name}/{extension[1:]}/{filename}" + return f"catalog/{catalog_name}/{extension[1:]}/{filename}" def _publication_block_directory_path(block: Any, filename: str) -> str: @@ -86,4 +89,4 @@ def _publication_block_directory_path(block: Any, filename: str) -> str: """ publication_id = block.publication_id unique_name = f"{uuid.uuid4().hex}_{filename}" - return f"files/public/publications/{publication_id}/{unique_name}" + return f"publications/{publication_id}/{unique_name}" diff --git a/authorization/migrations/0005_profile_picture_storage.py b/authorization/migrations/0005_profile_picture_storage.py new file mode 100644 index 00000000..61d9507e --- /dev/null +++ b/authorization/migrations/0005_profile_picture_storage.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0.4 on 2026-10-12 20:25 + +from django.db import migrations, models + +import api.utils.file_paths +import api.utils.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ("authorization", "0004_userconsent"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="profile_picture", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._user_profile_image_directory_path, + ), + ), + ] diff --git a/authorization/models.py b/authorization/models.py index 7a295002..85927475 100644 --- a/authorization/models.py +++ b/authorization/models.py @@ -3,6 +3,7 @@ from django.utils.text import slugify from api.utils.file_paths import _user_profile_image_directory_path +from api.utils.storage import public_storage from authorization.consent import UserConsent @@ -46,7 +47,10 @@ class User(AbstractUser): # Additional user profile fields bio = models.TextField(blank=True, null=True) profile_picture = models.ImageField( - upload_to=_user_profile_image_directory_path, blank=True, null=True + upload_to=_user_profile_image_directory_path, + storage=public_storage, + blank=True, + null=True, ) github_profile = models.URLField(blank=True, null=True) linkedin_profile = models.URLField(blank=True, null=True) From 85e183368d591552762d08fd7e23f20d8c0ca9d7 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:10:57 +0530 Subject: [PATCH 10/55] fix(api): read and serve media without local file paths S3 has no FieldFile.path, so every reader that used it would raise NotImplementedError once media left local disk. - local_copy() yields a temporary local copy (keeping the extension) for pandas, libmagic and the YAML/XML parsers; data indexing, entry counts and MIME validation use it. - The lru_cache on the loaders is dropped: it keyed on temp paths that are deleted on exit, so it only held DataFrames in memory. - Downloads stream with FileResponse instead of reading whole objects. - Charts only check that the file exists; they read the indexed table. - Search documents index the storage URL instead of a rewritten disk path. --- api/schema/resource_schema.py | 4 +- api/types/type_resource_chart.py | 9 ++-- api/utils/data_indexing.py | 58 +++++++++++----------- api/utils/file_utils.py | 43 ++++++++++------ api/views/download_view.py | 20 +++++--- api/views/publication_download_view.py | 9 ++-- search/documents/collaborative_document.py | 4 +- search/documents/usecase_document.py | 2 +- 8 files changed, 86 insertions(+), 63 deletions(-) diff --git a/api/schema/resource_schema.py b/api/schema/resource_schema.py index bb3c6d2a..143fd637 100644 --- a/api/schema/resource_schema.py +++ b/api/schema/resource_schema.py @@ -1,4 +1,3 @@ -import copy import uuid from enum import Enum @@ -135,8 +134,7 @@ def _validate_file_details_and_update_format(resource: Resource) -> None: raise ValueError("Resource has no file details") file = file_details.file - deep_copy_file = copy.deepcopy(file) - mime_type = file_validation(deep_copy_file, file, FORMAT_MAPPING) + mime_type = file_validation(file, file, FORMAT_MAPPING) if not mime_type: raise ValueError("Unsupported file format.") diff --git a/api/types/type_resource_chart.py b/api/types/type_resource_chart.py index c4236ac5..17593365 100644 --- a/api/types/type_resource_chart.py +++ b/api/types/type_resource_chart.py @@ -1,7 +1,9 @@ # mypy: disable-error-code="valid-type" import json + from pyecharts.globals import CurrentConfig + CurrentConfig.ONLINE_HOST = "file:///code/" import uuid from datetime import datetime @@ -20,7 +22,6 @@ from api.types.type_dataset import TypeDataset from api.types.type_resource import TypeResource, TypeResourceSchema from api.utils.enums import ChartStatus, ChartTypes -from api.utils.file_utils import load_csv logger = structlog.get_logger(__name__) T = TypeVar("T", bound="TypeResourceChart") @@ -41,8 +42,10 @@ def chart_base(chart_details: ResourceChartDetails) -> Optional[Chart]: logger.error("invalid resource format") return None - data = load_csv(file_details.file.path) - except (AttributeError, FileNotFoundError): + # Charts read the indexed data table, not the file; only require that it exists. + if not file_details.file.storage.exists(file_details.file.name): + return None + except AttributeError: return None chart_class = CHART_REGISTRY.get(chart_details.chart_type) diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 8d56da54..16201045 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -9,7 +9,7 @@ from api.models.Resource import Resource, ResourceDataTable from api.models.ResourceSchema import ResourceSchema from api.types.type_preview_data import PreviewData -from api.utils.file_utils import load_tabular_data +from api.utils.file_utils import load_tabular_data, local_copy logger = structlog.get_logger("dataspace.data_indexing") @@ -173,16 +173,19 @@ def handler(signum: int, frame: Any) -> None: signal.alarm(0) signal.signal(signal.SIGALRM, original_handler) - # Apply timeout only on Unix-based systems where signal.SIGALRM is available - try: - with timeout(60): # 60 second timeout for loading data - df = load_tabular_data(file_details.file.path, format) - except TimeoutError as te: - logger.error(f"Timeout while loading data for resource {resource_id}: {str(te)}") - return None - except Exception: - # Fallback without timeout if signal.SIGALRM is not available (e.g., on Windows) - df = load_tabular_data(file_details.file.path, format) + with local_copy(file_details.file) as file_path: + # Apply timeout only on Unix-based systems where signal.SIGALRM is available + try: + with timeout(60): # 60 second timeout for loading data + df = load_tabular_data(file_path, format) + except TimeoutError as te: + logger.error( + f"Timeout while loading data for resource {resource_id}: {str(te)}" + ) + return None + except Exception: + # Fallback without timeout if signal.SIGALRM is not available (e.g., on Windows) + df = load_tabular_data(file_path, format) if df is None: logger.error( @@ -410,29 +413,28 @@ def get_entry_count_from_file(resource: Resource) -> int: return 0 fmt = file_details.format.lower() - filepath = file_details.file.path - - if fmt in ("yml", "yaml"): - import yaml - - with open(filepath, "r") as f: - data = yaml.safe_load(f) - if isinstance(data, list): - return len(data) - if isinstance(data, dict): - for v in data.values(): - if isinstance(v, list): - return len(v) - return len(data) + if fmt not in FILE_COUNT_FORMATS: return 0 - if fmt == "xml": + with local_copy(file_details.file) as filepath: + if fmt in ("yml", "yaml"): + import yaml + + with open(filepath, "r") as f: + data = yaml.safe_load(f) + if isinstance(data, list): + return len(data) + if isinstance(data, dict): + for v in data.values(): + if isinstance(v, list): + return len(v) + return len(data) + return 0 + import xml.etree.ElementTree as ET root = ET.parse(filepath).getroot() return len(list(root)) - - return 0 except Exception as e: logger.error(f"Error counting entries from file for resource {resource.id}: {str(e)}") return 0 diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py index 9028f799..b3ae56a3 100644 --- a/api/utils/file_utils.py +++ b/api/utils/file_utils.py @@ -1,8 +1,9 @@ import mimetypes import os -from functools import lru_cache +import tempfile +from contextlib import contextmanager from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Iterator, Optional, Union import magic import pandas as pd @@ -16,18 +17,36 @@ def check_ext(file_object: Any) -> Optional[str]: return ext_type +@contextmanager +def local_copy(fieldfile: Any) -> Iterator[str]: + """Yield the path of a temporary local copy of a stored file. + + Media lives in S3, which has no local path, but pandas, libmagic and the + YAML/XML parsers need one. The copy keeps the file's extension (some + readers sniff it) and is removed on exit, even if the caller raises. + """ + suffix = os.path.splitext(fieldfile.name)[1] + with tempfile.NamedTemporaryFile(suffix=suffix) as tmp: + fieldfile.open("rb") + try: + for chunk in fieldfile.chunks(): + tmp.write(chunk) + finally: + fieldfile.close() + tmp.flush() + yield tmp.name + + def check_mime_type(file_object: Any) -> Any: # Prefer from_file: libmagic 5.46 misidentifies zip buffers as # application/octet-stream via from_buffer, while from_file works. - file_path = getattr(file_object, "path", None) - if file_path: - return magic.from_file(file_path, mime=True) + if hasattr(file_object, "storage"): + with local_copy(file_object) as file_path: + return magic.from_file(file_path, mime=True) return magic.from_buffer(file_object.read(), mime=True) -def file_validation( - file_object: Any, ext_object: Any, mapping: Dict[str, str] -) -> Optional[Any]: +def file_validation(file_object: Any, ext_object: Any, mapping: Dict[str, str]) -> Optional[Any]: ext_type = check_ext(ext_object) print("ext--", ext_type) mime_type = check_mime_type(file_object) @@ -78,7 +97,6 @@ def normalize_path(path: Union[str, Path]) -> str: return os.path.normpath(str(path)) -@lru_cache() def load_csv(filepath: str) -> pd.DataFrame: """Load CSV file into a pandas DataFrame. @@ -91,7 +109,6 @@ def load_csv(filepath: str) -> pd.DataFrame: return pd.read_csv(filepath) -@lru_cache() def load_excel(filepath: str) -> pd.DataFrame: """Load Excel file into a pandas DataFrame. @@ -104,7 +121,6 @@ def load_excel(filepath: str) -> pd.DataFrame: return pd.read_excel(filepath) -@lru_cache() def load_ods(filepath: str) -> pd.DataFrame: """Load ODS file into a pandas DataFrame. @@ -117,7 +133,6 @@ def load_ods(filepath: str) -> pd.DataFrame: return pd.read_excel(filepath, engine="odf") -@lru_cache() def load_parquet(filepath: str) -> pd.DataFrame: """Load Parquet file into a pandas DataFrame. @@ -130,7 +145,6 @@ def load_parquet(filepath: str) -> pd.DataFrame: return pd.read_parquet(filepath) -@lru_cache() def load_feather(filepath: str) -> pd.DataFrame: """Load Feather file into a pandas DataFrame. @@ -143,7 +157,6 @@ def load_feather(filepath: str) -> pd.DataFrame: return pd.read_feather(filepath) -@lru_cache() def load_json(filepath: str) -> pd.DataFrame: """Load JSON file into a pandas DataFrame. @@ -156,7 +169,6 @@ def load_json(filepath: str) -> pd.DataFrame: return pd.read_json(filepath) -@lru_cache() def load_tsv(filepath: str) -> pd.DataFrame: """Load TSV file into a pandas DataFrame. @@ -169,7 +181,6 @@ def load_tsv(filepath: str) -> pd.DataFrame: return pd.read_csv(filepath, sep="\t") -@lru_cache() def load_tabular_data(filepath: str, format: str) -> Optional[pd.DataFrame]: """Load tabular data from various file formats into a pandas DataFrame. diff --git a/api/views/download_view.py b/api/views/download_view.py index 5d106483..d6668bae 100644 --- a/api/views/download_view.py +++ b/api/views/download_view.py @@ -6,7 +6,14 @@ from asgiref.sync import sync_to_async from django.core.exceptions import ObjectDoesNotExist from django.core.files.uploadedfile import UploadedFile -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse +from django.http import ( + FileResponse, + HttpRequest, + HttpResponse, + HttpResponseRedirect, + JsonResponse, +) +from django.http.response import HttpResponseBase from pyecharts.charts.chart import Chart from pyecharts.render import make_snapshot from selenium import webdriver @@ -46,7 +53,7 @@ def get_chart_image(id: uuid.UUID) -> ResourceChartImage: @sync_to_async def get_resource_response( resource: Resource, request: Optional[HttpRequest] = None -) -> HttpResponse: +) -> HttpResponseBase: """Get file response for a resource.""" # Link-only (platform-imported) resources: we hold no bytes, send the # user to the file on the source platform. Still counts as a download. @@ -62,7 +69,8 @@ def get_resource_response( if not file_details or not file_details.file: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse(file_details.file.read(), content_type="application/octet-stream") + # Stream from storage instead of reading the whole object into memory. + response = FileResponse(file_details.file.open("rb"), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"resource_{resource.name}.csv" @@ -98,12 +106,12 @@ def _track_download(resource: Resource, request: Optional[HttpRequest]) -> None: @sync_to_async -def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: +def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponseBase: """Get file response for a chart image.""" if not chart_image.image: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse(chart_image.image.read(), content_type="application/octet-stream") + response = FileResponse(chart_image.image.open("rb"), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"chart_{chart_image.id}.png" @@ -119,7 +127,7 @@ def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: return response -async def download(request: HttpRequest, type: str, id: uuid.UUID) -> HttpResponse: +async def download(request: HttpRequest, type: str, id: uuid.UUID) -> HttpResponseBase: """Handle download requests for resources, chart images, and charts.""" try: if type == "resource": diff --git a/api/views/publication_download_view.py b/api/views/publication_download_view.py index deac2435..8f076909 100644 --- a/api/views/publication_download_view.py +++ b/api/views/publication_download_view.py @@ -15,7 +15,8 @@ from typing import Any from django.db.models import F -from django.http import HttpRequest, HttpResponse, JsonResponse +from django.http import FileResponse, HttpRequest, JsonResponse +from django.http.response import HttpResponseBase from api.models import Publication, PublicationBlock from api.services.publication_blocks import can_access_block_file @@ -25,7 +26,7 @@ _INLINE_CONTENT_TYPES = {".pdf": "application/pdf"} -def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponse: +def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponseBase: """Serve a block's file if the caller may see its resource, else 404.""" # Find the block and its parent resource. try: @@ -52,7 +53,7 @@ def publication_block_download(request: HttpRequest, block_id: Any) -> HttpRespo return _build_file_response(block) -def _build_file_response(block: PublicationBlock) -> HttpResponse: +def _build_file_response(block: PublicationBlock) -> FileResponse: """Build the HTTP file response with the right content type and disposition.""" stored_name = block.file.name or "" extension = os.path.splitext(stored_name)[1].lower() @@ -60,6 +61,6 @@ def _build_file_response(block: PublicationBlock) -> HttpResponse: disposition = "inline" if extension in _INLINE_CONTENT_TYPES else "attachment" filename = block.file_name or os.path.basename(stored_name) - response = HttpResponse(block.file.read(), content_type=content_type) + response = FileResponse(block.file.open("rb"), content_type=content_type) response["Content-Disposition"] = f'{disposition}; filename="{filename}"' return response diff --git a/search/documents/collaborative_document.py b/search/documents/collaborative_document.py index ef1ec70f..1fedaca3 100644 --- a/search/documents/collaborative_document.py +++ b/search/documents/collaborative_document.py @@ -246,12 +246,12 @@ def prepare_organizations(self, instance: Collaborative) -> List[Dict[str, str]] def prepare_logo(self, instance: Collaborative) -> str: if instance.logo: - return str(instance.logo.path.replace("/code/files/", "")) + return str(instance.logo.url) return "" def prepare_cover_image(self, instance: Collaborative) -> str: if instance.cover_image: - return str(instance.cover_image.path.replace("/code/files/", "")) + return str(instance.cover_image.url) return "" def prepare_dataset_count(self, instance: Collaborative) -> int: diff --git a/search/documents/usecase_document.py b/search/documents/usecase_document.py index b2b9e023..9849d753 100644 --- a/search/documents/usecase_document.py +++ b/search/documents/usecase_document.py @@ -226,7 +226,7 @@ def prepare_organizations(self, instance: UseCase) -> List[Dict[str, str]]: def prepare_logo(self, instance: UseCase) -> str: """Prepare logo URL for indexing.""" if instance.logo: - return str(instance.logo.path.replace("/code/files/", "")) + return str(instance.logo.url) return "" def should_index_object(self, obj: UseCase) -> bool: From 1eb56a3602bfa27d0fdc3887901303187e2af8a7 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:11:03 +0530 Subject: [PATCH 11/55] feat(api): redirect legacy /public media links to S3 Links issued before the move pointed at /public/ on disk. That route now redirects to the file's public S3 URL, after stripping the old "files/public/" or "files/" prefix from the name. Only public files redirect. The old static() route also served resources and drafts whenever DEBUG was on; anything outside public storage, or a name that tries to traverse paths, is a 404 here. --- DataSpace/urls.py | 16 +++++++++++----- api/views/legacy_media.py | 36 ++++++++++++++++++++++++++++++++++++ search/urls.py | 2 +- 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 api/views/legacy_media.py diff --git a/DataSpace/urls.py b/DataSpace/urls.py index 281b717e..85f149c5 100644 --- a/DataSpace/urls.py +++ b/DataSpace/urls.py @@ -23,7 +23,7 @@ from drf_yasg import openapi from drf_yasg.views import get_schema_view -from api.views import health +from api.views import health, legacy_media # Type alias for URL patterns URLPatternsList = List[Union[URLPattern, URLResolver]] @@ -49,16 +49,22 @@ # Health check endpoint path("health/", health.health_check, name="health_check"), # API documentation - path( - "swagger/", schema_view.without_ui(cache_timeout=0), name="schema-json" - ), + path("swagger/", schema_view.without_ui(cache_timeout=0), name="schema-json"), path( "swagger/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui", ), path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"), -] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + # Links issued before media moved to S3; see api/views/legacy_media.py. + path( + # Django prefixes a relative MEDIA_URL with the script name ("/public/"), + # which a URL pattern never matches. + f"{settings.MEDIA_URL.lstrip('/')}", + legacy_media.legacy_media_redirect, + name="legacy_media", + ), +] # In debug mode, add static URL patterns and debug toolbar if settings.DEBUG: diff --git a/api/views/legacy_media.py b/api/views/legacy_media.py new file mode 100644 index 00000000..e8900a24 --- /dev/null +++ b/api/views/legacy_media.py @@ -0,0 +1,36 @@ +""" +legacy_media +──────────── +Keeps links issued before the move to S3 working. Media used to be served from +disk at ``/public/``; that path now redirects to the file's S3 URL. + +Names are normalised first: rows created on disk carried a ``files/public/`` or +``files/`` prefix that S3 keys no longer have, so ``/public/files/public/x.png`` +redirects to the file now stored as ``x.png``. + +Only public files redirect. The old route also exposed resources and draft +files whenever DEBUG was on; anything not in public storage is a 404 here, so +this route can never hand out a private object. +""" + +import posixpath + +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse + +from api.utils.storage import normalize_media_name, public_storage + + +def legacy_media_redirect(request: HttpRequest, name: str) -> HttpResponse: + """Redirect an old ``/public/`` link to the public file's S3 URL, else 404.""" + normalised = posixpath.normpath(name) + if normalised != name or normalised.startswith(("/", "..")): + return JsonResponse({"error": "Not found"}, status=404) + + storage = public_storage() + current = normalize_media_name(name) + if not storage.exists(current): + return JsonResponse({"error": "Not found"}, status=404) + + response = HttpResponseRedirect(storage.url(current)) + response["Cache-Control"] = "public, max-age=86400" + return response diff --git a/search/urls.py b/search/urls.py index 7e663078..0b8ad4c4 100644 --- a/search/urls.py +++ b/search/urls.py @@ -32,4 +32,4 @@ # urlpatterns = [ # path('', include(router.urls)), -# ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +# ] From a1ef6f9c18b280e06369e3337edaddc05c5e30d1 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:11:03 +0530 Subject: [PATCH 12/55] feat(api): add migrate_media_to_s3 command Rows created on disk carry a "files/public/" or "files/" prefix that the S3 keys no longer use. For each stored name the command puts the file at the normalised key in the storage its field names, taken from the old S3 key or, with --source, from the old disk tree, then repoints the row. Dry run by default; --execute copies and renames. Safe to re-run (names already in place are skipped) and deletes nothing. --- .../commands/migrate_media_to_s3.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 api/management/commands/migrate_media_to_s3.py diff --git a/api/management/commands/migrate_media_to_s3.py b/api/management/commands/migrate_media_to_s3.py new file mode 100644 index 00000000..a85bf0d7 --- /dev/null +++ b/api/management/commands/migrate_media_to_s3.py @@ -0,0 +1,144 @@ +"""Bring every stored file into S3 under its current, prefix-free name. + + python manage.py migrate_media_to_s3 # dry run + python manage.py migrate_media_to_s3 --execute # copy + rename + python manage.py migrate_media_to_s3 --source /code/files/public --execute + +Rows created on disk carry a ``files/public/`` or ``files/`` prefix that the S3 +keys no longer use (see ``normalize_media_name``). For each stored name this +puts the file at the normalised key in the storage its field names, taking it +from the old S3 key or, with ``--source``, from the old disk tree, then points +the row at the new name. Safe to re-run: names already in place are skipped. +Nothing is deleted, neither local files nor old S3 objects. +""" + +from collections import Counter +from pathlib import Path +from typing import Any, Iterator, List, Optional, Set, Tuple, Type + +import structlog +from django.apps import apps +from django.core.files import File +from django.core.files.storage import Storage +from django.core.management.base import BaseCommand, CommandError, CommandParser +from django.db import models + +from api.utils.storage import normalize_media_name + +logger = structlog.get_logger("dataspace.commands.migrate_media_to_s3") + + +def iter_file_fields() -> Iterator[Tuple[Type[models.Model], models.FileField]]: + """Yield every concrete file field across installed models.""" + for model in apps.get_models(): + for field in model._meta.get_fields(): + if isinstance(field, models.FileField): + yield model, field + + +def stored_names(model: Type[models.Model], field: models.FileField) -> List[str]: + """Distinct non-empty names stored in ``field``.""" + return list( + model._default_manager.exclude(**{field.name: ""}) + .exclude(**{f"{field.name}__isnull": True}) + .values_list(field.name, flat=True) + .distinct() + ) + + +def copy_to(storage: Storage, target: str, fh: Any) -> None: + """Save ``fh`` under exactly ``target``; the row will point at that name.""" + saved = storage.save(target, File(fh)) + if saved != target: + storage.delete(saved) + raise RuntimeError(f"storage renamed {target} to {saved}") + + +class Command(BaseCommand): + help = "Copy media into S3 under prefix-free names and point DB rows at them." + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument( + "--source", + help="Old MEDIA_ROOT that legacy names are relative to (e.g. /code/files/public)", + ) + parser.add_argument( + "--execute", action="store_true", help="Copy and rename; without it, only report." + ) + + def handle(self, *args: Any, **options: Any) -> None: + source: Optional[Path] = None + if options["source"]: + source = Path(options["source"]).resolve() + if not source.is_dir(): + raise CommandError(f"Source directory not found: {source}") + execute: bool = options["execute"] + + counts: Counter = Counter() + seen: Set[str] = set() + for model, field in iter_file_fields(): + storage = field.storage + for name in stored_names(model, field): + target = normalize_media_name(name) + seen.add(target) + try: + present = self._place(storage, name, target, source, execute, counts) + if present and target != name: + counts["rows_renamed"] += 1 + if execute: + model._default_manager.filter(**{field.name: name}).update( + **{field.name: target} + ) + except Exception as exc: + counts["failed"] += 1 + logger.error("media_migration_failed", name=name, error=str(exc)) + + if source: + counts["orphaned"] = sum( + 1 + for p in source.rglob("*") + # Disk files keep their legacy names; rows may already use the clean ones. + if p.is_file() + and normalize_media_name(p.relative_to(source).as_posix()) not in seen + ) + + summary = ", ".join(f"{key}={counts[key]}" for key in sorted(counts)) + logger.info("media_migration_summary", execute=execute, **dict(counts)) + self.stdout.write(f"{'Done' if execute else 'Dry run'}: {summary}") + if counts["failed"]: + raise CommandError(f"{counts['failed']} file(s) failed") + + def _place( + self, + storage: Storage, + name: str, + target: str, + source: Optional[Path], + execute: bool, + counts: Counter, + ) -> bool: + """Make sure ``target`` exists in ``storage``; return whether it does (or would).""" + if storage.exists(target): + counts["in_place"] += 1 + return True + + if target != name and storage.exists(name): + counts["moved_in_s3"] += 1 + if execute: + with storage.open(name, "rb") as fh: + copy_to(storage, target, fh) + return True + + if source is not None: + local = (source / name).resolve() + # Names come from the DB; never read outside the source tree. + if local.is_relative_to(source) and local.is_file(): + counts["copied_from_disk"] += 1 + if execute: + with local.open("rb") as fh: + copy_to(storage, target, fh) + return True + + counts["missing"] += 1 + logger.warning("media_source_missing", name=name) + return False From df06b6a97ecd24b196af6bdab62527d4396b6159 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:11:17 +0530 Subject: [PATCH 13/55] test(api): cover the move to S3-only media Regression tests for the local-disk/DVC to S3 migration: S3 storage options (unsigned public URLs, 5-minute presigned private URLs, private default), which storage each file field names, local_copy() and MIME checks on storage without paths, prefix-free upload names, the legacy /public redirect (public files only, no traversal), createMajorVersion's owner-only version write, and migrate_media_to_s3 (dry run, disk and old S3 sources, missing files, failed copies). --- tests/test_s3_storage.py | 290 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 tests/test_s3_storage.py diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py new file mode 100644 index 00000000..27c0ce58 --- /dev/null +++ b/tests/test_s3_storage.py @@ -0,0 +1,290 @@ +"""Regression tests for the move from local disk (and DVC) to S3-only media. + +Root cause: uploads lived on local disk under MEDIA_ROOT and several code paths +read them through ``FieldFile.path`` (data indexing, upload MIME validation, +search documents, DVC versioning). S3 has no local path, so each of those +would raise ``NotImplementedError`` once media moved to a bucket. DVC never +worked at all: files were never under ``DVC_REPO_PATH``, so every DVC call +raised and ``createMajorVersion`` always failed. + +Fix: media goes through two S3 storages (``public``/``private``, keys prefixed +by ``STORAGE_ENV``); readers that need a real file use ``local_copy()``; old +``/public/`` links redirect to public files only; DVC is removed and +``createMajorVersion`` writes the version row directly behind a permission +check. Stored names drop the ``files/public/``/``files/`` prefix left from the +disk layout (S3 keys already sit under ``/public``); ``migrate_media_to_s3`` +moves existing files to those names and repoints the rows. +""" + +import io +import types +from pathlib import Path +from unittest.mock import patch + +import pytest +from django.apps import apps +from django.core.files.base import ContentFile +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import models +from django.test import RequestFactory +from django.urls import resolve +from storages.backends.s3 import S3Storage + +from api.models import Dataset, Resource, ResourceFileDetails, ResourceVersion +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.file_paths import ( + _chart_image_directory_path, + _organization_directory_path, + _publication_block_directory_path, +) +from api.utils.file_utils import check_mime_type, local_copy +from api.utils.storage import normalize_media_name, private_storage, public_storage +from api.views.legacy_media import legacy_media_redirect +from authorization.models import User +from DataSpace import settings as project_settings + +FAKE_KEYS = {"access_key": "AKIAEXAMPLEKEY000000", "secret_key": "not-a-real-secret"} + + +def _s3(options): + # URL building and presigning are local operations; no request reaches AWS. + return S3Storage(**{**options, **FAKE_KEYS}) + + +class TestS3StorageOptions: + def test_public_urls_are_unsigned_under_env_public_prefix(self): + url = _s3(project_settings._S3_PUBLIC_OPTIONS).url("organizations/a/logo.png") + assert url.endswith(f"/{project_settings.STORAGE_ENV}/public/organizations/a/logo.png") + assert "X-Amz-Signature" not in url + + def test_private_urls_are_presigned_for_five_minutes(self): + url = _s3(project_settings._S3_PRIVATE_OPTIONS).url("resources/data.csv") + assert f"/{project_settings.STORAGE_ENV}/private/resources/data.csv" in url + assert "X-Amz-Signature" in url + assert "X-Amz-Expires=300" in url + + def test_default_storage_is_private(self): + assert project_settings.STORAGES["default"] == project_settings.STORAGES["private"] + + def test_s3_has_no_local_path(self): + with pytest.raises(NotImplementedError): + _s3(project_settings._S3_PRIVATE_OPTIONS).path("resources/data.csv") + + +# A new file field that forgets storage= silently lands in the private default, +# so a public image would 403. Keep this list in step with the models. +PRIVATE_FILE_FIELDS = { + ("api", "resourcefiledetails", "file"), + ("api", "resourcechartimage", "image"), +} + + +class TestFileFieldStorages: + def _file_fields(self): + for model in apps.get_models(): + for field in model._meta.get_fields(): + if isinstance(field, models.FileField): + yield (model._meta.app_label, model._meta.model_name, field.name), field + + def test_every_file_field_names_public_or_private_storage(self): + for key, field in self._file_fields(): + assert field._storage_callable in (public_storage, private_storage), key + + def test_only_resource_files_and_chart_images_are_private(self): + private = { + key for key, field in self._file_fields() if field._storage_callable is private_storage + } + assert private == PRIVATE_FILE_FIELDS + + +@pytest.mark.django_db +class TestLocalCopy: + def _stored(self, name, content): + dataset = Dataset.objects.create(title="Local copy") + resource = Resource.objects.create(dataset=dataset, name="r", slug=f"r-{name}") + details = ResourceFileDetails(resource=resource, format="csv") + details.file.save(name, ContentFile(content), save=False) + return details.file + + def test_yields_a_local_file_with_the_stored_bytes_and_extension(self): + fieldfile = self._stored("data.csv", b"a,b\n1,2\n") + with local_copy(fieldfile) as path: + assert path.endswith(".csv") + assert Path(path).read_bytes() == b"a,b\n1,2\n" + + def test_removes_the_temp_file_even_when_the_caller_raises(self): + fieldfile = self._stored("boom.csv", b"x") + with pytest.raises(RuntimeError): + with local_copy(fieldfile) as path: + raise RuntimeError("boom") + assert not Path(path).exists() + + def test_mime_check_reads_a_stored_file_without_a_path(self): + fieldfile = self._stored("report.pdf", b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + assert check_mime_type(fieldfile) == "application/pdf" + + +class TestLegacyMediaRedirect: + def _get(self, name): + return legacy_media_redirect(RequestFactory().get(f"/public/{name}"), name) + + def test_route_points_at_the_redirect_view(self): + assert resolve("/public/files/public/x.png").func is legacy_media_redirect + + def test_public_file_redirects_to_its_storage_url(self): + name = public_storage().save("organizations/acme/png/logo.png", ContentFile(b"png")) + response = self._get(name) + assert response.status_code == 302 + assert response["Location"] == public_storage().url(name) + + def test_old_prefixed_link_redirects_to_the_renamed_file(self): + name = public_storage().save("organizations/old/png/logo.png", ContentFile(b"png")) + response = self._get(f"files/public/{name}") + assert response.status_code == 302 + assert response["Location"] == public_storage().url(name) + + def test_private_file_is_never_redirected(self): + name = private_storage().save("resources/secret.csv", ContentFile(b"a,b")) + assert self._get(name).status_code == 404 + + @pytest.mark.parametrize("name", ["../etc/passwd", "files/../../etc/passwd", "/etc/passwd"]) + def test_traversal_is_rejected(self, name): + assert self._get(name).status_code == 404 + + +MAJOR_VERSION = """ +mutation($input: CreateMajorVersionInput!) { + createMajorVersion(input: $input) { ... on TypeResource { id } } +} +""" + + +@pytest.mark.django_db +class TestCreateMajorVersionWithoutDvc: + @pytest.fixture + def owner(self): + return User.objects.create(username="owner", keycloak_id="owner") + + @pytest.fixture + def resource(self, owner): + dataset = Dataset.objects.create(title="Versioned", user=owner) + return Resource.objects.create(dataset=dataset, name="data", slug="data-versioned") + + def _run(self, user, resource): + return schema.execute_sync( + MAJOR_VERSION, + variable_values={"input": {"resourceId": str(resource.id), "description": "schema"}}, + context_value=types.SimpleNamespace(user=user, context={}), + ) + + def test_owner_gets_a_version_row_and_bump(self, owner, resource): + result = self._run(owner, resource) + assert result.errors is None, result.errors + resource.refresh_from_db() + assert resource.version == "v1.0.0" + assert ResourceVersion.objects.filter(resource=resource, version_number="v1.0.0").exists() + + def test_a_stranger_cannot_version_someone_elses_resource(self, resource): + stranger = User.objects.create(username="stranger", keycloak_id="stranger") + result = self._run(stranger, resource) + assert result.errors + assert not ResourceVersion.objects.filter(resource=resource).exists() + + +class TestNameLayout: + def test_upload_names_carry_no_legacy_prefix(self): + org = types.SimpleNamespace(name="Acme") + chart = types.SimpleNamespace(name="Rain") + block = types.SimpleNamespace(publication_id="p1") + assert _organization_directory_path(org, "logo.png") == "organizations/Acme/png/logo.png" + assert _chart_image_directory_path(chart, "c.png") == "chart_images/Rain/png/c.png" + assert _publication_block_directory_path(block, "r.pdf").startswith("publications/p1/") + + @pytest.mark.parametrize( + "old, new", + [ + ("files/public/organizations/a/png/l.png", "organizations/a/png/l.png"), + ("files/use_case/Health/logo/l.png", "use_case/Health/logo/l.png"), + ("resources/data.csv", "resources/data.csv"), + ], + ) + def test_normalize_media_name(self, old, new): + assert normalize_media_name(old) == new + + +@pytest.mark.django_db +class TestMigrateMediaToS3: + OLD = "files/public/organizations/acme/png/logo.png" + NEW = "organizations/acme/png/logo.png" + + @pytest.fixture + def storage(self): + storage = Organization._meta.get_field("logo").storage + for name in (self.OLD, self.NEW): + if storage.exists(name): + storage.delete(name) + return storage + + @pytest.fixture + def source(self, tmp_path): + logo = tmp_path / self.OLD + logo.parent.mkdir(parents=True) + logo.write_bytes(b"logo-bytes") + (tmp_path / "stray.txt").write_text("no row points here") + return tmp_path + + @pytest.fixture + def org(self, storage): + return Organization.objects.create(name="Acme", slug="acme", logo=self.OLD) + + def _run(self, *args): + out = io.StringIO() + call_command("migrate_media_to_s3", *args, stdout=out) + return out.getvalue() + + def _read(self, storage, name): + with storage.open(name) as fh: + return fh.read() + + def test_dry_run_writes_nothing(self, source, storage, org): + output = self._run("--source", str(source)) + assert "copied_from_disk=1" in output and "rows_renamed=1" in output + assert not storage.exists(self.NEW) + org.refresh_from_db() + assert org.logo.name == self.OLD + + def test_disk_file_lands_under_the_clean_name_and_row_follows(self, source, storage, org): + assert "copied_from_disk=1" in self._run("--source", str(source), "--execute") + assert self._read(storage, self.NEW) == b"logo-bytes" + org.refresh_from_db() + assert org.logo.name == self.NEW + assert "in_place=1" in self._run("--source", str(source), "--execute") + + def test_old_s3_key_is_copied_to_the_clean_name(self, storage, org): + storage.save(self.OLD, ContentFile(b"from-s3")) + assert "moved_in_s3=1" in self._run("--execute") + assert self._read(storage, self.NEW) == b"from-s3" + assert storage.exists(self.OLD) # nothing is deleted + org.refresh_from_db() + assert org.logo.name == self.NEW + + def test_counts_missing_and_orphaned_files(self, source, storage, org): + Organization.objects.create(name="Gone", slug="gone", logo="files/public/gone.png") + output = self._run("--source", str(source)) + assert "missing=1" in output + assert "orphaned=1" in output + + def test_a_missing_file_keeps_its_row_name(self, storage): + gone = Organization.objects.create(name="Gone", slug="gone", logo="files/public/gone.png") + self._run("--execute") + gone.refresh_from_db() + assert gone.logo.name == "files/public/gone.png" + + def test_a_failed_copy_exits_non_zero(self, source, storage, org): + with patch.object(storage, "save", side_effect=OSError("denied")): + with pytest.raises(CommandError): + self._run("--source", str(source), "--execute") + org.refresh_from_db() + assert org.logo.name == self.OLD From 062eed7bcf62023ce234ff5377760258039d9b60 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:11:40 +0530 Subject: [PATCH 14/55] docs: describe S3-only media storage Replace the DVC references in SECURITY.md and AGENTS.md with the S3 storage layout and access rules, and add the no-co-author commit rule to AGENTS.md. --- AGENTS.md | 5 +++-- SECURITY.md | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 78f32026..01684c19 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,8 @@ Before you call a change finished: - Cache: Redis (`django-redis`, `django-cacheops`) - Activity feed: `django-activity-stream` - Observability: OpenTelemetry, `structlog` -- Data: pandas, pyarrow, DVC for dataset versioning +- Data: pandas, pyarrow +- Media: S3 only, via `django-storages` (`public`/`private` storages in `api/utils/storage.py`; keys prefixed by `STORAGE_ENV`) - Runtime: uvicorn/gunicorn in Docker, deployed to AWS (ECS / EC2 via `scripts/ci-*.sh`) **File structure:** @@ -51,7 +52,6 @@ Before you call a change finished: - `views/`: REST views (search, downloads, charts, health, auth) - `services/`: business logic kept out of resolvers (publication, platform import, metadata mapping/export, model clients) - `activities/`: activity-stream handlers and display per domain (`dataset.py`, `organization.py`, …), wired up in `registry.py` - - `managers/`: `dvc_manager.py` for dataset versioning - `signals/`: post-save hooks (search indexing, cache invalidation) - `middleware/`, `utils/middleware.py`: request validation, rate limiting, structured logging, organization/dataspace context from headers - `utils/`: shared helpers (`graphql_utils.py`, `keycloak_utils.py`, `search_query.py`, `search_cache.py`, `enums.py`, …) @@ -158,6 +158,7 @@ def update_org(self, info, input): - Branch from `dev` with `fix/…`, `feat/…` or `chore/…`. PRs go into `dev`. Releases go from `dev` to `main`, gated by `pr-gate.yml`. - Do not make any commits utill asked for it. - Use Conventional Commits with a scope: `fix(api): …`, `feat(usecase): …`. +- Never use claude co-author while doing commit. - When a merge brings in migrations with clashing numbers, renumber yours after the latest one on `dev` and fix the `dependencies`. ## Boundaries diff --git a/SECURITY.md b/SECURITY.md index e3caa67a..f3afb381 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -134,21 +134,20 @@ When contributing to this project, please follow these security guidelines: - **GraphQL**: Implements query complexity limits and depth restrictions - **REST API**: Rate limiting and throttling enabled - **File Uploads**: Validated for type, size, and content -- **Data Versioning**: DVC integration for data versioning and tracking ### Data Protection - **Encryption**: All data encrypted in transit (TLS 1.2+) - **Database**: PostgreSQL with proper access controls - **Search**: Elasticsearch with authentication enabled - **Caching**: Redis with secure configuration -- **File Storage**: Secure file handling and validation +- **File Storage**: Media in S3; public assets via unsigned URLs, private files via presigned URLs or gated views ### Third-Party Services - **Keycloak**: Ensure proper realm and client configuration - **Elasticsearch**: Enable security features and authentication - **Redis**: Use password protection and disable dangerous commands - **OpenTelemetry**: Configure to exclude sensitive data -- **DVC**: Secure data versioning and storage +- **S3**: Block public access except the `*/public/*` prefix; least-privilege task role ## Security Checklist for Pull Requests From fa89bfa70595760581fca46aa4d93c06f4d54797 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Mon, 19 Oct 2026 12:17:21 +0530 Subject: [PATCH 15/55] docs: expand AGENTS.md with setup, checklists and media rules Agents kept tripping over setup the file never mentioned (.env and logs/ are needed before tests boot; pre-commit pins python3.11) and over S3-era rules that lived only in code review. Add a quick start, a commands table with the SDK test command, task checklists for mutations, model changes and bug fixes, a files-and-media section (no FieldFile.path, local_copy, FileResponse, public vs private storage), and git safety rules (self-contained commits, no AI co-author trailer, never stash the user's work). Existing rules are unchanged. --- AGENTS.md | 241 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 172 insertions(+), 69 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 01684c19..3f7f4bea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,40 +1,71 @@ # AGENTS.md -You are a senior engineer on the DataSpace backend. +Instructions for AI coding agents (and humans) working on the DataSpace backend. +Read this file first; the rules here override general habits. -## Persona +You are a senior engineer on this codebase: - You write clean, reusable, maintainable code that reads like the code around it. -- You understand the codebase and its security risks (authorization, multi-tenant organization data, file uploads, search exposure), and you turn that into clear, actionable insights. -- Your output: small, well-tested changes and reviews that a teammate can merge with confidence, plus a short explanation of *why* when the reason isn't obvious. +- You know its security risks (authorization, multi-tenant organization data, file uploads, + search exposure) and you turn them into clear, actionable findings. +- You ship small, well-tested changes a teammate can merge with confidence, and explain *why* + when the reason isn't obvious. + +## Quick start + +```bash +cp .env.example .env # settings fail without it (e.g. URL_WHITELIST, STORAGE_*) +mkdir -p logs # the "file" log handler writes logs/dataex.log +pip install -r requirements.txt +pip install pytest pytest-django # test-only, not in requirements.txt +``` + +- **Run (Docker, hot reload):** `docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build` +- **Run (venv):** `python manage.py runserver` ## Commands -- **Run locally (Docker, hot reload):** `docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build` -- **Run locally (venv):** copy `.env.example` to `.env`, then `python manage.py runserver` -- **Test:** `pytest tests/` uses SQLite in-memory with `--nomigrations` (see `pytest.ini`). It needs `pytest pytest-django`, which are not in `requirements.txt`. CI runs it with `--ignore=tests/object_types/charts --deselect tests/schema/test_dataset_schema.py::TestAddUpdateDatasetMetadata` (known-broken, see #189). -- **Single test:** `pytest tests/test_slug_collision.py -k usecase` -- **Format:** `black .` and `isort --profile black .` (line length 100, set in `pyproject.toml`; that file is the SDK's packaging config, but black reads it for the whole repo) -- **Type check:** `mypy --config-file=mypy.ini .` -- **All hooks:** `pre-commit run --all-files` (whitespace, black, isort, mypy, `debug-statements`) -- **Migrations:** `python manage.py makemigrations api`, then `python manage.py migrate` -- **Search mappings:** `python manage.py sync_search_mappings` after changing anything in `search/documents/` -- **API testing:** Postman CLI is installed; collections live in `postman/collections/` + +| Task | Command | +|---|---| +| Tests (as CI runs them) | `pytest tests/ --ignore=tests/object_types/charts --deselect tests/schema/test_dataset_schema.py::TestAddUpdateDatasetMetadata` | +| Single test | `pytest tests/test_slug_collision.py -k usecase` | +| SDK tests (no Django) | `pytest tests/test_auth.py tests/test_base.py tests/test_client.py tests/test_datasets.py tests/test_aimodels.py tests/test_usecases.py tests/test_exceptions.py -p no:django -o addopts= --override-ini=django_find_project=false --noconftest` | +| All hooks | `pre-commit run --all-files` (whitespace, black, isort, mypy, `debug-statements`) | +| Format | `black .` and `isort --profile black .` (line length 100, from `pyproject.toml`) | +| Type check | `mypy --config-file=mypy.ini .` | +| Migrations | `python manage.py makemigrations api`, then `python manage.py migrate` | +| Search mappings | `python manage.py sync_search_mappings` after any change in `search/documents/` | +| API testing | Postman CLI; collections live in `postman/collections/` | + +Notes: +- Tests use SQLite in memory with `--nomigrations` (`pytest.ini`, settings in + `tests/test_settings.py`). All media storages are `InMemoryStorage`, so tests never reach S3. +- The CI exclusions above are known-broken tests (#189). Don't add to them. +- pre-commit hooks pin `python3.11`. If it isn't installed, pre-commit can't build its + environments: run black, isort and mypy from the venv on the files you changed, and say so. +- mypy results depend on its cache. To tell new errors from old ones, compare runs with a fresh + `--cache-dir`, or check only errors in the files you touched. ## Definition of done -Before you call a change finished: -1. `pre-commit run --all-files` is clean. -2. The tests pass with the CI flags above, and a bug fix has a regression test. -3. Every new or changed GraphQL query/mutation has the right permission guard (see Rules). -4. If a model changed: its search document, signals, cache invalidation and SDK still match. + +A change is finished only when: +1. `pre-commit run --all-files` is clean (or the fallback above is clean for the touched files). +2. Tests pass with the CI flags, and a bug fix has a regression test. +3. Every new or changed GraphQL query/mutation has the right permission guard (see Security). +4. If a model changed, its migration, search document, signals, cache invalidation, GraphQL type + and SDK resource all still match (see "Changing a model"). ## Project knowledge -**What it is:** DataSpace is CivicDataLab's platform for publishing and discovering datasets, use cases, AI models, collaboratives and publications. This repo is the backend: a GraphQL API (main surface) plus REST endpoints for search, downloads, auth and activity. It also ships a Python client, `dataspace_sdk`, published to PyPI. +**What it is:** DataSpace is CivicDataLab's platform for publishing and discovering datasets, +use cases, AI models, collaboratives and publications. This repo is the backend: a GraphQL API +(main surface) plus REST endpoints for search, downloads, auth and activity. It also ships a +Python client, `dataspace_sdk`, published to PyPI. **Tech stack:** - Python 3.11, Django 5.0, PostgreSQL - GraphQL: `strawberry-graphql` 0.235 + `strawberry-graphql-django` 0.42 (endpoint `/api/graphql`) - REST: Django REST Framework 3.14, `drf-yasg` for API docs -- Auth: Keycloak (`python-keycloak` 5.5); every request needs a real Keycloak JWT, with no dev-mode fallback +- Auth: Keycloak (`python-keycloak` 5.5). Every request needs a real Keycloak JWT; there is no dev-mode fallback. - Search: Elasticsearch 8 via `django-elasticsearch-dsl` - Cache: Redis (`django-redis`, `django-cacheops`) - Activity feed: `django-activity-stream` @@ -43,58 +74,121 @@ Before you call a change finished: - Media: S3 only, via `django-storages` (`public`/`private` storages in `api/utils/storage.py`; keys prefixed by `STORAGE_ENV`) - Runtime: uvicorn/gunicorn in Docker, deployed to AWS (ECS / EC2 via `scripts/ci-*.sh`) -**File structure:** +**Layout:** - `DataSpace/`: Django project (`settings.py`, `urls.py`, `asgi.py`, `cache_settings.py`) - `api/`: main app - - `models/`: one model per file (`Dataset.py`, `Organization.py`, `UseCase.py`, …), exported from `models/__init__.py` - - `schema/`: Strawberry `Query`/`Mutation` classes per domain (`*_schema.py`), stitched together in `schema/schema.py`. `base_mutation.py` holds `BaseMutation`/`MutationResponse`. `extensions.py` holds `TrackActivity`/`TrackModelActivity`. + - `models/`: one model per file (`Dataset.py`, `UseCase.py`, …), exported from `models/__init__.py` + - `schema/`: Strawberry `Query`/`Mutation` classes per domain (`*_schema.py`), merged with + `merge_types` in `schema/schema.py`. `base_mutation.py` holds `BaseMutation`/`MutationResponse`; + `extensions.py` holds `TrackActivity`/`TrackModelActivity`. - `types/`: Strawberry output types (`type_*.py`) - - `views/`: REST views (search, downloads, charts, health, auth) + - `views/`: REST views (search, downloads, charts, health, auth, legacy media redirect) - `services/`: business logic kept out of resolvers (publication, platform import, metadata mapping/export, model clients) - - `activities/`: activity-stream handlers and display per domain (`dataset.py`, `organization.py`, …), wired up in `registry.py` - - `signals/`: post-save hooks (search indexing, cache invalidation) + - `activities/`: activity-stream handlers and display per domain, wired up in `registry.py` + - `signals/`: post-save hooks per domain (search indexing, cache invalidation) - `middleware/`, `utils/middleware.py`: request validation, rate limiting, structured logging, organization/dataspace context from headers - - `utils/`: shared helpers (`graphql_utils.py`, `keycloak_utils.py`, `search_query.py`, `search_cache.py`, `enums.py`, …) - - `management/commands/`: `sync_search_mappings`, `update_dataindex`, `seed_resource_types`, `populate_geographies`, … + - `utils/`: shared helpers (`graphql_utils.py`, `keycloak_utils.py`, `storage.py`, `file_utils.py`, `file_paths.py`, `search_query.py`, `search_cache.py`, `enums.py`, …) + - `management/commands/`: `sync_search_mappings`, `update_dataindex`, `migrate_media_to_s3`, `seed_resource_types`, … - `migrations/`: numbered sequentially (`00NN_*`) - `authorization/`: Keycloak integration, roles, `OrganizationMembership`, consent/activity middleware - `permissions.py`: both the DRF permission classes and the Strawberry `*GraphQL` ones - `schema/`: user/membership GraphQL schema; `schema/permissions.py` holds `HasOrganizationAdminRole` - `graphql_permissions.py`: legacy duplicate of the GraphQL classes; don't add new uses - `search/documents/`: Elasticsearch document definitions and analysers -- `dataspace_sdk/`: standalone Python SDK (its own version in `pyproject.toml`; depends only on `requests`) -- `tests/`: pytest suite (`test_*.py`, `tests/schema/`, `tests/journeys/`), settings in `tests/test_settings.py` +- `dataspace_sdk/`: standalone Python SDK (`client.py`, `resources/.py`; own version in `pyproject.toml`; depends only on `requests`) +- `tests/`: pytest suite (`test_*.py`, `tests/schema/`, `tests/journeys/`). `test_auth.py`, `test_base.py`, `test_client.py`, `test_datasets.py`, `test_aimodels.py`, `test_usecases.py` and `test_exceptions.py` test the SDK. - `docs/`: feature docs (Keycloak, unified search, charts, activity stream, SDK) -- `postman/`: Postman collections and fixtures for API testing +- `postman/`: Postman collections and fixtures - `aws/`, `scripts/`, `.github/workflows/`: infra, deploy scripts, CI -- Root-level `migrate_*.py`, `populate_sdgs.py`, `test_geography_hierarchy.py`: one-off scripts. Leave them alone, and put new code in `api/management/commands/` instead. - -## Standards - -**Naming conventions:** -- Functions, methods, variables: snake_case (`create_organization`, `get_current_user`) -- Classes: PascalCase (`OrganizationInput`, `HasOrganizationRole`) -- Constants: UPPER_SNAKE_CASE (`MAX_RETRIES`) -- Strawberry output types: `Type` in `api/types/type_.py` -- Model files: PascalCase, one model per file (`api/models/UseCase.py`) -- GraphQL fields are snake_case in Python. Strawberry exposes them as camelCase (`create_organization` → `createOrganization`). - -**Rules:** -- Type-annotate every function. mypy runs in pre-commit, so don't add new `# type: ignore` or `mypy.ini` exemptions without a reason in a comment. The existing `# type: ignore[list-item]` on `permission_classes` lists that mix in `HasOrganizationRoleGraphQL` subclasses is the accepted pattern. - - mypy skips `tests/` and is relaxed for `api.models`, `api.activities` and the resource, use case and dataset schemas (see `mypy.ini`). Still annotate new code there. -- Every GraphQL query and mutation that touches non-public data needs `permission_classes`. Don't rely on the ORM to reject anonymous users. - - Import from `authorization.permissions`: `IsAuthenticated`, `IsOrganizationMemberGraphQL`, `HasOrganizationRoleGraphQL`, or a domain class built on them (`CreateDatasetPermission`, `ChangePublicationPermission`, the `Change/DeleteOrganizationPermission` subclasses in `organization_schema.py`, …). - - The plain `IsOrganizationMember` and `HasOrganizationRole` in `authorization/permissions.py` are DRF classes for REST views. They don't work in Strawberry `permission_classes`. Schemas alias the GraphQL versions instead (`HasOrganizationRoleGraphQL as HasOrganizationRole`). - - `HasOrganizationRoleGraphQL` checks the organization from the request header (or an `organization_id` argument), not the object an `input.id` points at. When a mutation loads an object by id, also check the caller's membership in that object's organization inside the resolver. -- Scope querysets to what the user can see (organization membership, dataspace, published status). Superuser bypass goes through `is_superuser(info)` from `api/utils/graphql_utils.py`. -- Keep resolvers thin. Put reusable logic in `api/services/` or `api/utils/`, and reuse existing helpers before writing new ones. +- Root-level `migrate_*.py`, `populate_sdgs.py`, `test_geography_hierarchy.py`: one-off scripts. + Leave them alone; new scripts go in `api/management/commands/`. + +## Code standards + +**Naming:** +- Functions, methods, variables: snake_case. Classes: PascalCase. Constants: UPPER_SNAKE_CASE. +- Strawberry output types: `Type` in `api/types/type_.py`. +- Model files: PascalCase, one model per file (`api/models/UseCase.py`). +- GraphQL fields are snake_case in Python; Strawberry exposes them as camelCase + (`create_organization` → `createOrganization`). + +**Style:** +- Type-annotate every function. Don't add `# type: ignore` or `mypy.ini` exemptions without a + reason in a comment. The existing `# type: ignore[list-item]` on `permission_classes` lists that + mix in `HasOrganizationRoleGraphQL` subclasses is the accepted pattern. + mypy skips `tests/` and is relaxed for `api.models`, `api.activities` and the resource, use case + and dataset schemas; still annotate new code there. +- Keep resolvers thin. Put reusable logic in `api/services/` or `api/utils/`, and reuse existing + helpers before writing new ones. - Wrap multi-step writes in `transaction.atomic()`. -- Return clean client errors (400/404, or `MutationResponse.error_response`) instead of letting `IntegrityError`, `ValueError` or `DoesNotExist` surface as a 500. -- Log with the existing logger and `structlog`, without tokens or PII. -- Comments explain *why* (constraints, past incidents), not *what*. Match the density of the surrounding file. -- Write regression tests in the style of `tests/test_slug_collision.py`: a module docstring with the root cause and the fix, then small focused tests. +- Return clean client errors (400/404, or `MutationResponse.error_response`). Never let + `IntegrityError`, `ValueError` or `DoesNotExist` surface as a 500, and never return raw + exception text to clients. +- Log with the module's existing `structlog` logger. Never log tokens, secrets or PII. +- Comments explain *why* (constraints, past incidents), not *what*. Match the density of the + surrounding file. +- Make the smallest change that solves the problem. Don't reformat, rename or refactor + unrelated code in the same change. + +## Security + +- Every GraphQL query and mutation that touches non-public data needs `permission_classes`. + Don't rely on the ORM to reject anonymous users. + - Import from `authorization.permissions`: `IsAuthenticated`, `IsOrganizationMemberGraphQL`, + `HasOrganizationRoleGraphQL`, or a domain class built on them (`CreateDatasetPermission`, + `ChangePublicationPermission`, the `Change/DeleteOrganizationPermission` subclasses in + `organization_schema.py`, …). + - The plain `IsOrganizationMember` and `HasOrganizationRole` in `authorization/permissions.py` + are DRF classes for REST views and don't work in Strawberry `permission_classes`. Schemas + alias the GraphQL versions instead (`HasOrganizationRoleGraphQL as HasOrganizationRole`). + - `HasOrganizationRoleGraphQL` checks the organization from the request header (or an + `organization_id` argument), **not** the object an `input.id` points at. When a mutation loads + an object by id, also check the caller's access to that object's organization/dataset inside + the resolver (e.g. `AuthorizationService.check_dataset_permission`). +- Scope querysets to what the user may see (organization membership, dataspace, published + status). Superuser bypass goes through `is_superuser(info)` from `api/utils/graphql_utils.py`. +- Search must not expose non-public objects. Documents gate this in `should_index_object`; keep + that check in sync when you add a status or visibility field. + +## Files and media + +- Media lives only in S3. Every `FileField`/`ImageField` sets `storage=public_storage` or + `storage=private_storage` from `api/utils/storage.py`. Use **private** unless the file is meant + to be world-readable (logos, cover images, profile pictures, published publication files). +- Never use `FieldFile.path`: S3 has no local path (and tests use `InMemoryStorage`). Use + `file.open("rb")`/`file.chunks()`, `file.storage.exists(file.name)`, or + `api.utils.file_utils.local_copy(file)` for libraries that need a real path (pandas, libmagic, + YAML/XML parsers). +- Serve downloads with `FileResponse(file.open("rb"))`, not `HttpResponse(file.read())`. +- `upload_to` helpers in `api/utils/file_paths.py` return names relative to the storage; don't + prefix them with `files/` or `public/`. +- Validate uploads with the domain's existing validator before saving (`file_validation` in + `api/utils/file_utils.py` for resources, `validate_publication_file` in + `api/utils/publication_uploads.py` for publication blocks). Don't trust the extension alone. + +## Task checklists + +**Adding or changing a GraphQL mutation** +1. Put it in the domain's `api/schema/_schema.py`; logic in `api/services/` if non-trivial. +2. Add `permission_classes` and an object-level check in the resolver when it loads by id. +3. Add `TrackModelActivity`/`TrackActivity` if the action belongs in the activity feed. +4. Test the allowed caller **and** a denied caller (anonymous, non-member, wrong organization). + +**Changing a model** (ask first: see Boundaries) +1. Edit `api/models/.py`, then `python manage.py makemigrations api` and review the file. +2. Update the `Type` in `api/types/` and any input types in the schema. +3. Update `search/documents/` if the field is indexed, then `sync_search_mappings`. +4. Check `api/signals/` and cache invalidation still cover the change. +5. Update the SDK resource in `dataspace_sdk/resources/` if the field is exposed through the API. + +**Fixing a bug** +1. Reproduce it with a failing test first. +2. Write the regression test in the style of `tests/test_slug_collision.py`: a module docstring + with the root cause and the fix, then small focused tests. +3. Fix the cause, not the symptom. + +## Code style example -**Code style example:** ```python from api.schema.extensions import TrackModelActivity from api.utils.graphql_utils import is_superuser @@ -154,26 +248,35 @@ def update_org(self, info, input): return org ``` -## Git & PRs -- Branch from `dev` with `fix/…`, `feat/…` or `chore/…`. PRs go into `dev`. Releases go from `dev` to `main`, gated by `pr-gate.yml`. -- Do not make any commits utill asked for it. -- Use Conventional Commits with a scope: `fix(api): …`, `feat(usecase): …`. -- Never use claude co-author while doing commit. -- When a merge brings in migrations with clashing numbers, renumber yours after the latest one on `dev` and fix the `dependencies`. +## Git and PRs + +- Don't commit until asked. +- Branch from `dev` with `fix/…`, `feat/…` or `chore/…`. PRs go into `dev`. Releases go from + `dev` to `main`, gated by `pr-gate.yml`. +- Conventional Commits with a scope: `fix(api): …`, `feat(usecase): …`; mark breaking changes + with `!` and a `BREAKING CHANGE:` footer. The body says *why*, not a list of files. +- Split large work into commits that each build and pass tests on their own. +- Never add a Claude (or any AI) `Co-Authored-By` trailer to commits. +- Never `git stash`, `reset` or `checkout --` the user's uncommitted work. To compare against a + baseline, use `git worktree add HEAD` (copy `.env` and create `logs/` there). +- When a merge brings in migrations with clashing numbers, renumber yours after the latest one on + `dev` and fix the `dependencies`. ## Boundaries -- ✅ **Always:** write code in `api/`, `authorization/`, `search/`, `dataspace_sdk/` and `tests/`. Run `pytest` and `pre-commit` before committing. + +- ✅ **Always:** work in `api/`, `authorization/`, `search/`, `dataspace_sdk/` and `tests/`. Run + the tests and hooks before saying a change is done. - ⚠️ **Ask first:** - - model or schema changes and new migrations + - model or schema changes, and new migrations - changes to permission classes, Keycloak/auth middleware or rate limits - changes to Elasticsearch mappings (these need a reindex) - - adding or upgrading dependencies + - adding, removing or upgrading dependencies - bumping the SDK version or publishing it - touching `.github/workflows/`, `aws/`, `scripts/ci-*.sh`, `Dockerfile` or `docker-compose.yml` - - anything that runs against a deployed environment + - anything that runs against a deployed environment or a real S3 bucket - 🚫 **Never:** - commit `.env`, secrets, Keycloak client secrets or tokens. Use `.env.example` for new variables. - edit or delete already-merged migrations. Add a new migration instead. - - disable an auth check or a failing test to get green. Fix the cause, or ask. + - disable an auth check, skip or delete a failing test to get green. Fix the cause, or ask. - edit `.venv/`, `files/` (user uploads) or `logs/` - force-push to `dev` or `main` From f8d747aa5b92a734deeb75af5017cdc7d47ad8fc Mon Sep 17 00:00:00 2001 From: psaesha Date: Wed, 21 Oct 2026 23:13:29 +0530 Subject: [PATCH 16/55] feat(metadata): add source identifier dataset metadata item CivicDataLab's scrapers fill "Source identifier" on every dataset they upload (dataspace-scrapers//) and read it back to recognise their own datasets, so a dataset is never created twice and their upload records can be rebuilt from DataSpace. The data migration creates the item, or switches it back on if an admin switched it off; reversing it deletes nothing. Tests: tests/test_source_identifier_migration.py Co-Authored-By: Claude Opus 5.5 --- .../0054_add_source_identifier_metadata.py | 48 ++++++++++++ tests/test_source_identifier_migration.py | 78 +++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 api/migrations/0054_add_source_identifier_metadata.py create mode 100644 tests/test_source_identifier_migration.py diff --git a/api/migrations/0054_add_source_identifier_metadata.py b/api/migrations/0054_add_source_identifier_metadata.py new file mode 100644 index 00000000..f39e0d8c --- /dev/null +++ b/api/migrations/0054_add_source_identifier_metadata.py @@ -0,0 +1,48 @@ +"""Add the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers (DataSpace-Scrapers) fill it on every dataset they upload with +dataspace-scrapers//, and read it back to recognise their own +datasets, so a dataset is never created twice and their upload records can be rebuilt +from DataSpace. People never fill it in. Platform imports (Kaggle, Hugging Face, GitHub) +also fill it, with the platform's own ID, because the label matches the import's +"source identifier" field; only values starting with dataspace-scrapers/ are a scraper's. + +Creates the item if it is missing and switches it back on if an admin switched it off. +Reversing it deletes nothing, since datasets may already carry values for it. +""" + +from django.db import migrations + +LABEL = "Source identifier" + + +def add_source_identifier(apps, schema_editor): + Metadata = apps.get_model("api", "Metadata") + item, created = Metadata.objects.get_or_create( + label=LABEL, + model="DATASET", + defaults={ + "urn": "dataspace-scrapers:source-identifier", + "data_type": "STRING", + "type": "OPTIONAL", + "data_standard": "NA", + "options": None, + "validator": [], + "enabled": True, + "filterable": False, + }, + ) + if not created and not item.enabled: + item.enabled = True + item.save(update_fields=["enabled"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.RunPython(add_source_identifier, migrations.RunPython.noop), + ] diff --git a/tests/test_source_identifier_migration.py b/tests/test_source_identifier_migration.py new file mode 100644 index 00000000..1f9d6e94 --- /dev/null +++ b/tests/test_source_identifier_migration.py @@ -0,0 +1,78 @@ +"""Tests for the migration that adds the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers look the item up by its exact label, on datasets, and +only while it is switched on, so these check that the migration leaves exactly +one such item behind, switched on, whatever state it finds. + +The test settings skip migrations, so the migration's function is called +directly against the current models. +""" + +import importlib + +import pytest +from django.apps import apps + +from api.models import Metadata + +migration = importlib.import_module("api.migrations.0054_add_source_identifier_metadata") + + +def source_identifier_items(): + return Metadata.objects.filter(label="Source identifier", model="DATASET") + + +@pytest.mark.django_db +def test_creates_the_item_switched_on_when_missing(): + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "dataspace-scrapers:source-identifier" + assert item.data_type == "STRING" + assert item.type == "OPTIONAL" + assert item.filterable is False + + +@pytest.mark.django_db +def test_switches_an_existing_item_back_on_without_adding_another(): + Metadata.objects.create( + label="Source identifier", + model="DATASET", + urn="admin-made:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "admin-made:source-identifier" + + +@pytest.mark.django_db +def test_running_twice_leaves_one_item(): + migration.add_source_identifier(apps, None) + migration.add_source_identifier(apps, None) + + assert source_identifier_items().count() == 1 + + +@pytest.mark.django_db +def test_leaves_a_same_named_item_on_another_model_alone(): + resource_item = Metadata.objects.create( + label="Source identifier", + model="RESOURCE", + urn="resource:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + resource_item.refresh_from_db() + assert resource_item.enabled is False + assert source_identifier_items().get().urn == "dataspace-scrapers:source-identifier" From a74e2e9d64d158d8c9456890ee5385873596f907 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 22 Oct 2026 17:28:05 +0530 Subject: [PATCH 17/55] feat(api): table queries for use cases, collaboratives, AI models, publications Add useCasesTable, collaborativesTable, aiModelsTable and publicationsTable, shaped like datasetsTable: client filters and sort options limited to an allowlist per module, limit/offset paging, the total after filters, and counts per status for the dashboard tabs (ignoring the status filter so tab labels stay stable). Each query shows the same rows as the module's existing list query, plus published (for AI models: public and active) rows when includePublic is set. The existing list queries are unchanged. StatusCount and status_counts() are shared in api/utils/qs_utils.py. No migration. --- api/schema/aimodel_schema.py | 95 +++++++++++++++++++++++++++ api/schema/collaborative_schema.py | 93 ++++++++++++++++++++++++++ api/schema/publication_schema.py | 84 ++++++++++++++++++++++++ api/schema/usecase_schema.py | 101 ++++++++++++++++++++++++++++- api/types/type_aimodel.py | 15 +++++ api/types/type_collaborative.py | 15 +++++ api/types/type_publication.py | 15 +++++ api/types/type_usecase.py | 15 +++++ api/utils/qs_utils.py | 26 +++++++- 9 files changed, 457 insertions(+), 2 deletions(-) diff --git a/api/schema/aimodel_schema.py b/api/schema/aimodel_schema.py index 0b059a45..abd76ae3 100644 --- a/api/schema/aimodel_schema.py +++ b/api/schema/aimodel_schema.py @@ -8,6 +8,7 @@ import strawberry import strawberry_django from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet from strawberry.types import Info from strawberry_django.pagination import OffsetPaginationInput @@ -21,6 +22,7 @@ AIModelLifecycleStageEnum, AIModelOrder, AIModelProviderEnum, + AIModelResponse, AIModelStatusEnum, AIModelTypeEnum, AIModelVersionFilter, @@ -34,6 +36,12 @@ TypeVersionProvider, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.graphql_permissions import IsAuthenticated from authorization.models import OrganizationMembership, Role @@ -281,6 +289,57 @@ class UpdateVersionProviderInput: config: Optional[strawberry.scalars.JSON] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +AI_MODELS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "display_name", + "status", + "model_type", + "provider", + "domain", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "created_at", + "updated_at", +] +AI_MODELS_TABLE_SORT_FIELDS = [ + "name", + "display_name", + "created_at", + "updated_at", + "status", + "model_type", + "provider", +] + + +def _visible_ai_models(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The AI models a caller may list. Same rules as `aiModels`.""" + organization = info.context.context.get("organization") + user = info.context.user + public = AIModel.objects.filter(is_public=True, is_active=True) + if organization: + queryset = AIModel.objects.filter(organization=organization) + elif not user.is_authenticated: + queryset = public + elif user.is_superuser: + queryset = AIModel.objects.all() + else: + queryset = AIModel.objects.filter(user=user, organization=None) + if include_public: + queryset = queryset | public + return queryset.distinct() + + @strawberry.type class Query: """Queries for AI Models.""" @@ -330,6 +389,42 @@ def ai_models( return TypeAIModel.from_django_list(list(queryset)) + @strawberry.field + @trace_resolver(name="ai_models_table", attributes={"component": "aimodel"}) + def ai_models_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> AIModelResponse: + """AI models for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `aiModels`, with the same visibility rules, plus + public, active models when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_ai_models(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=AI_MODELS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created_at", direction="desc")], + sorting_allowed_fields=AI_MODELS_TABLE_SORT_FIELDS, + ) + return AIModelResponse( + data=TypeAIModel.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, AI_MODELS_TABLE_FILTER_FIELDS), + ) + @strawberry.field @trace_resolver(name="get_ai_model", attributes={"component": "aimodel"}) def get_ai_model(self, info: Info, model_id: int) -> Optional[TypeAIModel]: diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 366f4aa6..4a7bed46 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -9,6 +9,7 @@ import strawberry import strawberry_django from django.db import models +from django.db.models import QuerySet from strawberry import auto from strawberry.file_uploads import Upload from strawberry.types import Info @@ -37,6 +38,7 @@ from api.types.type_collaborative import ( CollaborativeFilter, CollaborativeOrder, + CollaborativeResponse, TypeCollaborative, ) from api.types.type_collaborative_organization import ( @@ -48,6 +50,12 @@ from api.types.type_usecase import TypeUseCase from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.models import User from authorization.types import TypeUser @@ -95,6 +103,55 @@ class CollaborativeInputPartial: completed_on: Optional[datetime.date] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +COLLABORATIVES_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "sdgs__code", + "sdgs__slug", + "started_on", + "completed_on", + "created", + "modified", +] +COLLABORATIVES_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "started_on", + "completed_on", +] + + +def _visible_collaboratives(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The collaboratives a caller may list. Same rules as `collaboratives`.""" + user = info.context.user + organization = info.context.context.get("organization") + if organization: + queryset = Collaborative.objects.filter(organization=organization) + elif user.is_superuser: + queryset = Collaborative.objects.all() + elif user.is_authenticated: + queryset = Collaborative.objects.filter(user=user) + else: + queryset = Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) + if include_public: + queryset = queryset | Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) + return queryset.distinct() + + @strawberry.type(name="Query") class Query: """Queries for collaboratives.""" @@ -138,6 +195,42 @@ def collaboratives( return TypeCollaborative.from_django_list(queryset) + @strawberry.field + @trace_resolver(name="collaboratives_table", attributes={"component": "collaborative"}) + def collaboratives_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> CollaborativeResponse: + """Collaboratives for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `collaboratives`, with the same visibility rules, plus + published collaboratives when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_collaboratives(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=COLLABORATIVES_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=COLLABORATIVES_TABLE_SORT_FIELDS, + ) + return CollaborativeResponse( + data=TypeCollaborative.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, COLLABORATIVES_TABLE_FILTER_FIELDS), + ) + @strawberry_django.field @trace_resolver(name="get_published_collaboratives", attributes={"component": "collaborative"}) def published_collaboratives( diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py index 8b0e354a..53eecd97 100644 --- a/api/schema/publication_schema.py +++ b/api/schema/publication_schema.py @@ -16,6 +16,7 @@ import strawberry import strawberry_django from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet from strawberry.file_uploads import Upload from strawberry.types import Info @@ -41,6 +42,7 @@ from api.types.type_publication import ( PublicationFilter, PublicationOrder, + PublicationResponse, TypePublication, TypePublicationBlock, TypeResourceType, @@ -48,6 +50,12 @@ ) from api.utils.enums import PublicationStatus from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.permissions import ( AllowPublishedPublications, ChangePublicationPermission, @@ -93,6 +101,46 @@ class UpdatePublicationInput: external_source_link: Optional[str] = strawberry.UNSET +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +PUBLICATIONS_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "license", + "resource_type", + "resource_type__slug", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "publication_date", + "created", + "modified", +] +PUBLICATIONS_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "publication_date", + "download_count", +] + + +def _visible_publications(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The publications a caller may list. Same rules as `publications`.""" + return get_scoped_publications( + user=info.context.user, + organization=info.context.context.get("organization"), + include_public=bool(include_public), + ) + + @strawberry.type(name="Query") class Query: """Queries for publications.""" @@ -147,6 +195,42 @@ def publications( ) return TypePublication.from_django_list(queryset[offset : offset + limit]) + @strawberry.field + @trace_resolver(name="publications_table", attributes={"component": "publication"}) + def publications_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> PublicationResponse: + """Publications for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `publications`, with the same visibility rules, plus + published publications when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_publications(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=PUBLICATIONS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=PUBLICATIONS_TABLE_SORT_FIELDS, + ) + return PublicationResponse( + data=TypePublication.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, PUBLICATIONS_TABLE_FILTER_FIELDS), + ) + @strawberry.type(name="Mutation") class Mutation: diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index 7d3c67fe..6b133d6a 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -10,6 +10,7 @@ import strawberry import strawberry_django from django.db import models +from django.db.models import QuerySet from strawberry import auto from strawberry.file_uploads import Upload from strawberry.types import Info @@ -36,7 +37,12 @@ ) from api.types.type_dataset import TypeDataset from api.types.type_organization import TypeOrganization -from api.types.type_usecase import TypeUseCase, UseCaseFilter, UseCaseOrder +from api.types.type_usecase import ( + TypeUseCase, + UseCaseFilter, + UseCaseOrder, + UseCaseResponse, +) from api.types.type_usecase_organization import ( TypeUseCaseOrganizationRelationship, relationship_type, @@ -47,6 +53,12 @@ UseCaseStatus, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.models import User from authorization.types import TypeUser @@ -95,6 +107,57 @@ class UseCaseInputPartial: completed_on: Optional[datetime.date] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +USE_CASES_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "running_status", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "sdgs__code", + "sdgs__slug", + "started_on", + "completed_on", + "created", + "modified", +] +USE_CASES_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "running_status", + "started_on", + "completed_on", +] + + +def _visible_use_cases(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The use cases a caller may list. Same rules as `useCases`.""" + user = info.context.user + organization = info.context.context.get("organization") + if organization: + queryset = UseCase.objects.filter(organization=organization) + elif user.is_superuser: + queryset = UseCase.objects.all() + elif user.is_authenticated: + queryset = UseCase.objects.filter(user=user) + else: + queryset = UseCase.objects.filter(status=UseCaseStatus.PUBLISHED) + if include_public: + queryset = queryset | UseCase.objects.filter(status=UseCaseStatus.PUBLISHED) + return queryset.distinct() + + @strawberry.type(name="Query") class Query: """Queries for use cases.""" @@ -138,6 +201,42 @@ def use_cases( return TypeUseCase.from_django_list(queryset) + @strawberry.field + @trace_resolver(name="use_cases_table", attributes={"component": "usecase"}) + def use_cases_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> UseCaseResponse: + """Use cases for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `useCases`, with the same visibility rules, plus + published use cases when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_use_cases(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=USE_CASES_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=USE_CASES_TABLE_SORT_FIELDS, + ) + return UseCaseResponse( + data=TypeUseCase.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, USE_CASES_TABLE_FILTER_FIELDS), + ) + @strawberry_django.field @trace_resolver(name="get_published_use_cases", attributes={"component": "usecase"}) def published_use_cases( diff --git a/api/types/type_aimodel.py b/api/types/type_aimodel.py index 60c7c7b7..e06baca6 100644 --- a/api/types/type_aimodel.py +++ b/api/types/type_aimodel.py @@ -30,6 +30,7 @@ HFModelClass, PromptDomain, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser logger = structlog.get_logger("dataspace.type_aimodel") @@ -334,3 +335,17 @@ def ai_model(self) -> Optional[TypeAIModel]: return TypeAIModel.from_django(django_instance.ai_model) except Exception: return None + + +@strawberry.type +class AIModelResponse: + """One page of a table listing (`aiModelsTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeAIModel] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_collaborative.py b/api/types/type_collaborative.py index aa624632..e5bb1d13 100644 --- a/api/types/type_collaborative.py +++ b/api/types/type_collaborative.py @@ -31,6 +31,7 @@ OrganizationRelationshipType, PublicationStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser collaborative_status = strawberry.enum(CollaborativeStatus) # type: ignore @@ -261,3 +262,17 @@ def partner_organizations(self) -> Optional[List["TypeOrganization"]]: return TypeOrganization.from_django_list(organizations) except Exception: return [] + + +@strawberry.type +class CollaborativeResponse: + """One page of a table listing (`collaborativesTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeCollaborative] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_publication.py b/api/types/type_publication.py index ee2781d1..cfb40a51 100644 --- a/api/types/type_publication.py +++ b/api/types/type_publication.py @@ -25,6 +25,7 @@ PublicationStatus, UseCaseStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser @@ -236,3 +237,17 @@ def linked_count(self, info: Info) -> int: return usecases + collabs except (AttributeError, Publication.DoesNotExist): return 0 + + +@strawberry.type +class PublicationResponse: + """One page of a table listing (`publicationsTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypePublication] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_usecase.py b/api/types/type_usecase.py index eb801ff6..60e8d6a0 100644 --- a/api/types/type_usecase.py +++ b/api/types/type_usecase.py @@ -30,6 +30,7 @@ PublicationStatus, UseCaseStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser use_case_status = strawberry.enum(UseCaseStatus) # type: ignore @@ -240,3 +241,17 @@ def usecase_dashboard(self) -> Optional[List["TypeUseCaseDashboard"]]: return TypeUseCaseDashboard.from_django_list(usecase_dashboards) except Exception: return [] + + +@strawberry.type +class UseCaseResponse: + """One page of a table listing (`useCasesTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeUseCase] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/utils/qs_utils.py b/api/utils/qs_utils.py index 5e6fc2c6..15846baf 100644 --- a/api/utils/qs_utils.py +++ b/api/utils/qs_utils.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Sequence import strawberry -from django.db.models import Q, QuerySet +from django.db.models import Count, Q, QuerySet MAX_LIMIT = 100 @@ -50,6 +50,12 @@ class SortSpec: direction: str = "asc" +@strawberry.type(description="Rows per status, for a table's tab labels.") +class StatusCount: + status: str + count: int + + def _check_allowed(kind: str, field: str, allowed: Optional[Sequence[str]]) -> None: if allowed is not None and field not in allowed: raise ValueError( @@ -128,3 +134,21 @@ def get_pagination_window( if sort_options: queryset = apply_sorting(queryset, sort_options, sorting_allowed_fields) return apply_pagination(queryset, limit, offset) + + +def status_counts( + queryset: QuerySet, + filters: Optional[Sequence[FilterSpec]] = None, + allowed_fields: Optional[Sequence[str]] = None, + status_field: str = "status", +) -> List[StatusCount]: + """Count rows per status under every filter except the status one, so tab + labels stay the same whichever tab is open.""" + others = [f for f in (filters or []) if (f.field or "").strip() != status_field] + rows = ( + apply_filters(queryset, others, allowed_fields) + .order_by() + .values(status_field) + .annotate(n=Count("pk", distinct=True)) + ) + return [StatusCount(status=row[status_field], count=row["n"]) for row in rows] From 4b57735497253f683b77ba498ffa3ec354f40c92 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 22 Oct 2026 20:17:41 +0530 Subject: [PATCH 18/55] refactor(api): datasetsTable uses the shared status_counts helper The tab counts in datasetsTable were an inline copy of what status_counts() in api/utils/qs_utils.py now does for the other four tables. Call the helper instead, so all five tables count statuses in one place. status_counts() takes an optional item_type so the dataset table keeps its DatasetStatusCount type; the GraphQL schema is unchanged. --- api/schema/dataset_schema.py | 16 ++++------------ api/utils/qs_utils.py | 10 ++++++---- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/api/schema/dataset_schema.py b/api/schema/dataset_schema.py index 40d89f12..2aab35a2 100644 --- a/api/schema/dataset_schema.py +++ b/api/schema/dataset_schema.py @@ -58,8 +58,8 @@ from api.utils.qs_utils import ( FilterSpec, SortSpec, - apply_filters, get_pagination_window, + status_counts, ) from authorization.models import DatasetPermission, OrganizationMembership, Role, User from authorization.permissions import ( @@ -539,20 +539,12 @@ def datasets_table( sort_options=sort_options or [SortSpec(field="created", direction="desc")], sorting_allowed_fields=DATASETS_TABLE_SORT_FIELDS, ) - # Tab labels: counts per status under every filter except status itself. - without_status = [f for f in (filters or []) if f.field.strip() != "status"] - status_rows = ( - apply_filters(visible, without_status, DATASETS_TABLE_FILTER_FIELDS) - .order_by() - .values("status") - .annotate(n=Count("id", distinct=True)) - ) return DatasetResponse( data=TypeDataset.from_django_list(window["data"]), total_items_count=window["total_items_count"], - status_counts=[ - DatasetStatusCount(status=row["status"], count=row["n"]) for row in status_rows - ], + status_counts=status_counts( + visible, filters, DATASETS_TABLE_FILTER_FIELDS, item_type=DatasetStatusCount + ), ) @strawberry.field( diff --git a/api/utils/qs_utils.py b/api/utils/qs_utils.py index 15846baf..803384ad 100644 --- a/api/utils/qs_utils.py +++ b/api/utils/qs_utils.py @@ -8,7 +8,7 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence, Type import strawberry from django.db.models import Count, Q, QuerySet @@ -141,9 +141,11 @@ def status_counts( filters: Optional[Sequence[FilterSpec]] = None, allowed_fields: Optional[Sequence[str]] = None, status_field: str = "status", -) -> List[StatusCount]: + item_type: Type[Any] = StatusCount, +) -> List[Any]: """Count rows per status under every filter except the status one, so tab - labels stay the same whichever tab is open.""" + labels stay the same whichever tab is open. ``item_type`` lets a module keep + its own GraphQL type name (``DatasetStatusCount``).""" others = [f for f in (filters or []) if (f.field or "").strip() != status_field] rows = ( apply_filters(queryset, others, allowed_fields) @@ -151,4 +153,4 @@ def status_counts( .values(status_field) .annotate(n=Count("pk", distinct=True)) ) - return [StatusCount(status=row[status_field], count=row["n"]) for row in rows] + return [item_type(status=row[status_field], count=row["n"]) for row in rows] From 6842cbfc55d7663d875eeb7250967724f682b708 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 27 Oct 2026 15:55:00 +0530 Subject: [PATCH 19/55] feat(api): add chartsTable with filters, sorting and pagination The charts dashboard loads every built chart and chart image through getChartData and searches them in the browser. chartsTable returns one page of both kinds in a single list, with the total count and per-status counts, using the shared table helpers. Both tables are merged in one SQL UNION, so counting, sorting and slicing happen in the database; only the rows of the page are then loaded. Visibility and permission are the same as getChartData. Rows keep their types (TypeResourceChart, TypeResourceChartImage), so existing fragments work unchanged. getChartData is untouched. --- api/schema/resource_chart_schema.py | 141 +++++++++++++++++++++++++++- api/types/type_resource_chart.py | 16 ++++ 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/api/schema/resource_chart_schema.py b/api/schema/resource_chart_schema.py index bf8d04da..1b51d234 100644 --- a/api/schema/resource_chart_schema.py +++ b/api/schema/resource_chart_schema.py @@ -6,17 +6,80 @@ import strawberry import strawberry_django +from django.db.models import CharField, QuerySet, Value from strawberry.types import Info from strawberry_django.mutations import mutations -from api.models import Resource, ResourceChartDetails, ResourceSchema +from api.models import ( + Resource, + ResourceChartDetails, + ResourceChartImage, + ResourceSchema, +) +from api.schema.dataset_schema import ChartDataPermission from api.schema.extensions import TrackModelActivity -from api.types.type_resource_chart import TypeResourceChart +from api.types.type_resource_chart import ChartResponse, TypeResourceChart +from api.types.type_resource_chart_image import TypeResourceChartImage from api.utils.enums import AggregateType, ChartStatus, ChartTypes +from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + StatusCount, + apply_filters, + apply_pagination, + apply_sorting, + status_counts, +) ChartTypeEnum = strawberry.enum(ChartTypes) AggregateTypeEnum = strawberry.enum(AggregateType) +# Fields chartsTable accepts. Names are the chart image's; a built chart reaches +# its dataset through its resource, so `dataset…` is rewritten for it. +# `kind` is CHART (built in the wizard) or IMAGE (uploaded). +CHARTS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "status", + "chart_type", + "kind", + "dataset", + "dataset__slug", + "dataset__title", + "modified", +] +CHARTS_TABLE_SORT_FIELDS = ["name", "status", "chart_type", "kind", "modified"] + +# Columns shared by both tables for the UNION. Model fields first, then the +# annotations, so both sides select them in the same order. +_CHART_COLUMNS = ("id", "name", "status", "modified", "chart_type", "kind") + + +def _for_built_charts(field: str) -> str: + if field == "dataset" or field.startswith("dataset__"): + return f"resource__{field}" + return field + + +def _visible_charts(info: Info) -> "tuple[QuerySet, QuerySet]": + """Built charts and chart images the caller may list: the same rules as + `getChartData` without a dataset (the organisation's, else the user's).""" + organization = info.context.context.get("organization") + user = info.context.user + if organization: + charts = ResourceChartDetails.objects.filter(resource__dataset__organization=organization) + images = ResourceChartImage.objects.filter(dataset__organization=organization) + else: + charts = ResourceChartDetails.objects.filter(resource__dataset__user=user) + images = ResourceChartImage.objects.filter(dataset__user=user) + charts = charts.annotate(kind=Value("CHART", output_field=CharField())) + images = images.annotate( + chart_type=Value(None, output_field=CharField()), + kind=Value("IMAGE", output_field=CharField()), + ) + return charts, images + @strawberry.type(name="Query") class Query: @@ -30,6 +93,80 @@ def resource_chart(self, info: Info, chart_details_id: uuid.UUID) -> TypeResourc chart = ResourceChartDetails.objects.get(id=chart_details_id) return TypeResourceChart.from_django(chart) + @strawberry.field( + permission_classes=[ChartDataPermission], # type: ignore[list-item] + ) + @trace_resolver(name="charts_table", attributes={"component": "chart"}) + def charts_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + ) -> ChartResponse: + """Built charts and chart images in one table: client-driven filters, + sortable columns and page controls backed by a real total count. + + Additive next to `getChartData`, with the same visibility rules. Both + tables are merged in one SQL UNION, so counting, sorting and slicing + happen in the database; only the rows of the page are then loaded. + Default order: last modified first. + """ + visible_charts, visible_images = _visible_charts(info) + # Images first: their field names are the public ones, so a disallowed + # field is reported with the allowlist as clients know it. + images = apply_filters(visible_images, filters or [], CHARTS_TABLE_FILTER_FIELDS) + chart_filters = [ + FilterSpec(field=_for_built_charts(f.field or ""), condition=f.condition, value=f.value) + for f in filters or [] + ] + chart_fields = [_for_built_charts(f) for f in CHARTS_TABLE_FILTER_FIELDS] + charts = apply_filters(visible_charts, chart_filters, chart_fields) + + # order_by() drops the models' default ordering, not allowed inside a UNION. + rows = ( + charts.order_by() + .values(*_CHART_COLUMNS) + .union(images.order_by().values(*_CHART_COLUMNS)) + ) + rows = apply_sorting( + rows, + sort_options or [SortSpec(field="modified", direction="desc")], + CHARTS_TABLE_SORT_FIELDS, + ) + # Tie-breaker so equal sort values do not move rows between pages. + rows = rows.order_by(*rows.query.order_by, "id") + window = apply_pagination(rows, limit, offset) + page = list(window["data"]) + + # from_django reads every foreign key of the chart's dataset. + built = ResourceChartDetails.objects.select_related( + "resource__dataset__organization", + "resource__dataset__user", + "resource__dataset__dataspace", + ).in_bulk([r["id"] for r in page if r["kind"] == "CHART"]) + uploaded = ResourceChartImage.objects.select_related("dataset").in_bulk( + [r["id"] for r in page if r["kind"] == "IMAGE"] + ) + data: List[Any] = [] + for r in page: + if r["kind"] == "CHART" and r["id"] in built: + data.append(TypeResourceChart.from_django(built[r["id"]])) + elif r["kind"] == "IMAGE" and r["id"] in uploaded: + data.append(TypeResourceChartImage.from_django(uploaded[r["id"]])) + + counts: Dict[str, int] = {} + for item in status_counts(visible_charts, chart_filters, chart_fields) + status_counts( + visible_images, filters, CHARTS_TABLE_FILTER_FIELDS + ): + counts[item.status] = counts.get(item.status, 0) + item.count + return ChartResponse( + data=data, + total_items_count=window["total_items_count"], + status_counts=[StatusCount(status=k, count=v) for k, v in sorted(counts.items())], + ) + @strawberry.input class FilterInput: diff --git a/api/types/type_resource_chart.py b/api/types/type_resource_chart.py index c4236ac5..59b7c353 100644 --- a/api/types/type_resource_chart.py +++ b/api/types/type_resource_chart.py @@ -19,8 +19,10 @@ from api.types.charts.chart_registry import CHART_REGISTRY from api.types.type_dataset import TypeDataset from api.types.type_resource import TypeResource, TypeResourceSchema +from api.types.type_resource_chart_image import TypeResourceChartImage from api.utils.enums import ChartStatus, ChartTypes from api.utils.file_utils import load_csv +from api.utils.qs_utils import StatusCount logger = structlog.get_logger(__name__) T = TypeVar("T", bound="TypeResourceChart") @@ -362,3 +364,17 @@ def chart(self, info: Info) -> Optional[ChartConfig]: height=chart_instance.height, renderer=chart_instance.renderer, ) + + +@strawberry.type +class ChartResponse: + """One page of the charts table (`chartsTable`): built charts and chart + images in one list, the total after filters, and per-status counts.""" + + data: List[Union[TypeResourceChartImage, TypeResourceChart]] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels do not change when switching tabs.", + ) From bf8e275eb8d31ce7fa865069985d8e2f9d2332ee Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 27 Oct 2026 16:48:53 +0530 Subject: [PATCH 20/55] refactor(api): page chartsTable through get_pagination_window Sorting and paging now go through the same helper as the other table queries. Only filtering stays per table, because Django cannot filter after a UNION. Drops the extra id tie-breaker, matching the others. --- api/schema/resource_chart_schema.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/schema/resource_chart_schema.py b/api/schema/resource_chart_schema.py index 1b51d234..a4e3b3b6 100644 --- a/api/schema/resource_chart_schema.py +++ b/api/schema/resource_chart_schema.py @@ -27,8 +27,7 @@ SortSpec, StatusCount, apply_filters, - apply_pagination, - apply_sorting, + get_pagination_window, status_counts, ) @@ -130,14 +129,14 @@ def charts_table( .values(*_CHART_COLUMNS) .union(images.order_by().values(*_CHART_COLUMNS)) ) - rows = apply_sorting( + # Filters are already applied per table above (not allowed after a UNION). + window = get_pagination_window( rows, - sort_options or [SortSpec(field="modified", direction="desc")], - CHARTS_TABLE_SORT_FIELDS, + limit=limit, + offset=offset, + sort_options=sort_options or [SortSpec(field="modified", direction="desc")], + sorting_allowed_fields=CHARTS_TABLE_SORT_FIELDS, ) - # Tie-breaker so equal sort values do not move rows between pages. - rows = rows.order_by(*rows.query.order_by, "id") - window = apply_pagination(rows, limit, offset) page = list(window["data"]) # from_django reads every foreign key of the chart's dataset. From 43d7eef6f9ab49a4d61c45849b087e4d5ef74115 Mon Sep 17 00:00:00 2001 From: Abhishekfm Date: Wed, 28 Oct 2026 15:45:55 +0530 Subject: [PATCH 21/55] feat(api): add Event model and GraphQL schema for event management This commit introduces a new Event model with fields for name, location, dates, and status, along with a corresponding GraphQL schema for creating, updating, and querying events. The schema includes input types for event creation and updates, as well as queries for retrieving events based on various filters. Additionally, utility functions for handling Luma embeds and event permissions are implemented to ensure proper validation and access control. --- api/migrations/0055_event.py | 77 ++++++ api/models/Event.py | 76 ++++++ api/models/__init__.py | 1 + api/schema/event_schema.py | 293 ++++++++++++++++++++++ api/schema/schema.py | 3 + api/services/event_service.py | 185 ++++++++++++++ api/types/type_event.py | 104 ++++++++ api/utils/enums.py | 5 + api/utils/luma.py | 175 +++++++++++++ authorization/permissions.py | 138 ++++++++++- tests/schema/test_event_schema.py | 393 ++++++++++++++++++++++++++++++ 11 files changed, 1448 insertions(+), 2 deletions(-) create mode 100644 api/migrations/0055_event.py create mode 100644 api/models/Event.py create mode 100644 api/schema/event_schema.py create mode 100644 api/services/event_service.py create mode 100644 api/types/type_event.py create mode 100644 api/utils/luma.py create mode 100644 tests/schema/test_event_schema.py diff --git a/api/migrations/0055_event.py b/api/migrations/0055_event.py new file mode 100644 index 00000000..61b8d1db --- /dev/null +++ b/api/migrations/0055_event.py @@ -0,0 +1,77 @@ +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0049_merge_publication_and_accessmodel"), + ("api", "0054_add_source_identifier_metadata"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Event", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("name", models.CharField(blank=True, max_length=300)), + ("slug", models.SlugField(max_length=255, unique=True)), + ("location", models.CharField(blank=True, max_length=300)), + ("start_date", models.DateField(blank=True, null=True)), + ("end_date", models.DateField(blank=True, null=True)), + ("luma_embed", models.JSONField(blank=True, default=dict)), + ( + "status", + models.CharField( + choices=[("DRAFT", "Draft"), ("PUBLISHED", "Published")], + default="DRAFT", + max_length=50, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="events", + to="api.organization", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="events", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "Event", + "verbose_name_plural": "Events", + "db_table": "event", + "ordering": ["-modified"], + "indexes": [ + models.Index( + fields=["organization", "-modified"], name="event_org_modified_idx" + ), + models.Index(fields=["user", "-modified"], name="event_user_modified_idx"), + models.Index(fields=["status"], name="event_status_idx"), + ], + }, + ), + ] diff --git a/api/models/Event.py b/api/models/Event.py new file mode 100644 index 00000000..d51c358e --- /dev/null +++ b/api/models/Event.py @@ -0,0 +1,76 @@ +import uuid +from typing import Any + +from django.db import models +from django.utils.text import slugify + +from api.utils.enums import EventStatus + + +class Event(models.Model): + """A Luma-backed event owned by an organization or an individual user. + + Peer to Publication: a UUID/slug entity with a draft/publish status. The + embed is stored as whitelisted iframe attributes, never as raw HTML. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=300, blank=True) + slug = models.SlugField(max_length=255, unique=True) + location = models.CharField(max_length=300, blank=True) + start_date = models.DateField(null=True, blank=True) + end_date = models.DateField(null=True, blank=True) + luma_embed = models.JSONField(default=dict, blank=True) + status = models.CharField( + max_length=50, + default=EventStatus.DRAFT, + choices=EventStatus.choices, + ) + + organization = models.ForeignKey( + "api.Organization", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="events", + ) + user = models.ForeignKey( + "authorization.User", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="events", + ) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + if not self.slug: + base_slug = slugify(self.name) or "untitled" + slug = base_slug + counter = 1 + while Event.objects.filter(slug=slug).exclude(pk=self.pk).exists(): + slug = f"{base_slug}-{counter}" + counter += 1 + self.slug = slug + super().save(*args, **kwargs) + + @property + def is_individual_event(self) -> bool: + """True when this event is owned by an individual, not an organization.""" + return self.organization is None and self.user is not None + + def __str__(self) -> str: + return self.name or "Untitled event" + + class Meta: + verbose_name = "Event" + verbose_name_plural = "Events" + db_table = "event" + ordering = ["-modified"] + indexes = [ + models.Index(fields=["organization", "-modified"], name="event_org_modified_idx"), + models.Index(fields=["user", "-modified"], name="event_user_modified_idx"), + models.Index(fields=["status"], name="event_status_idx"), + ] diff --git a/api/models/__init__.py b/api/models/__init__.py index 19f7c515..ebd4861a 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -11,6 +11,7 @@ from api.models.DatasetMetadata import DatasetMetadata from api.models.DatasetSource import DatasetSource from api.models.DataSpace import DataSpace +from api.models.Event import Event from api.models.Geography import Geography from api.models.Metadata import Metadata from api.models.Organization import Organization diff --git a/api/schema/event_schema.py b/api/schema/event_schema.py new file mode 100644 index 00000000..5ca45383 --- /dev/null +++ b/api/schema/event_schema.py @@ -0,0 +1,293 @@ +"""Schema definitions for events. + +Resolvers delegate metadata checks, writes, and scoping to +``api/services/event_service.py``. +""" + +import datetime +import uuid +from typing import List, Optional + +import strawberry +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet +from strawberry.types import Info + +from api.models import Event +from api.schema.base_mutation import BaseMutation, MutationResponse +from api.services.event_service import ( + apply_event_update, + assert_ready_to_publish, + create_event, + get_scoped_events, + set_event_status, +) +from api.types.type_event import EventResponse, TypeEvent +from api.utils.enums import EventStatus +from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) +from authorization.permissions import ( + AllowPublishedEvents, + ChangeEventPermission, + CreateEventPermission, + DeleteEventPermission, + PublishEventPermission, +) + + +@strawberry.input +class CreateEventInput: + """Fields for a new draft. Every field is optional.""" + + name: Optional[str] = None + location: Optional[str] = None + start_date: Optional[datetime.date] = None + end_date: Optional[datetime.date] = None + luma_embed: Optional[str] = None + + +@strawberry.input +class UpdateEventInput: + """Partial edit. ``UNSET`` means the client omitted the field.""" + + id: uuid.UUID + name: Optional[str] = strawberry.UNSET + location: Optional[str] = strawberry.UNSET + start_date: Optional[datetime.date] = strawberry.UNSET + end_date: Optional[datetime.date] = strawberry.UNSET + luma_embed: Optional[str] = strawberry.UNSET + + +EVENTS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "location", + "status", + "start_date", + "end_date", + "organization", + "organization__slug", + "created", + "modified", +] +EVENTS_TABLE_SORT_FIELDS = [ + "name", + "location", + "status", + "start_date", + "end_date", + "created", + "modified", +] + + +def _visible_events(info: Info) -> QuerySet: + return get_scoped_events( + user=info.context.user, + organization=info.context.context.get("organization"), + ) + + +@strawberry.type(name="Query") +class Query: + @strawberry.field(permission_classes=[AllowPublishedEvents]) # type: ignore[list-item] + @trace_resolver(name="get_event", attributes={"component": "event"}) + def event(self, info: Info, event_id: uuid.UUID) -> Optional[TypeEvent]: + """Get one event. Drafts are gated to the owner or organization.""" + try: + return TypeEvent.from_django(Event.objects.get(id=event_id)) + except Event.DoesNotExist: + return None + + @strawberry.field + @trace_resolver(name="events_table", attributes={"component": "event"}) + def events_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + ) -> EventResponse: + """Events for the dashboard table, scoped like the other providers.""" + visible = _visible_events(info) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=EVENTS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="modified", direction="desc")], + sorting_allowed_fields=EVENTS_TABLE_SORT_FIELDS, + ) + return EventResponse( + data=TypeEvent.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, EVENTS_TABLE_FILTER_FIELDS), + ) + + @strawberry.field + @trace_resolver(name="published_events", attributes={"component": "event"}) + def published_events( + self, + info: Info, + organization_slug: Optional[str] = None, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 20, + offset: int = 0, + ) -> EventResponse: + """Published events for public pages. Drafts are never included.""" + visible = Event.objects.filter(status=EventStatus.PUBLISHED).select_related( + "organization", "user" + ) + if organization_slug: + visible = visible.filter(organization__slug=organization_slug) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=EVENTS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="start_date", direction="asc")], + sorting_allowed_fields=EVENTS_TABLE_SORT_FIELDS, + ) + return EventResponse( + data=TypeEvent.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=[], + ) + + @strawberry.field + @trace_resolver(name="event_by_slug", attributes={"component": "event"}) + def event_by_slug(self, info: Info, slug: str) -> Optional[TypeEvent]: + """Published event for a public page. Drafts and unknown slugs return null.""" + event = ( + Event.objects.filter(slug=slug, status=EventStatus.PUBLISHED) + .select_related("organization", "user") + .first() + ) + if event is None: + return None + return TypeEvent.from_django(event) + + +@strawberry.type(name="Mutation") +class Mutation: + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[CreateEventPermission], + trace_name="create_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "created", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def create_event(self, info: Info, input: CreateEventInput) -> MutationResponse[TypeEvent]: + """Create a draft owned by the caller's organization or the user.""" + event = create_event( + user=info.context.user, + organization=info.context.context.get("organization"), + name=input.name or "", + location=input.location or "", + start_date=input.start_date, + end_date=input.end_date, + luma_embed=input.luma_embed, + ) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangeEventPermission], + trace_name="update_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "updated", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def update_event(self, info: Info, input: UpdateEventInput) -> MutationResponse[TypeEvent]: + """Apply a partial edit. Completeness is required only after publish.""" + event = _get_event_or_raise(input.id) + event = apply_event_update(event, **_update_kwargs(input)) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishEventPermission], + trace_name="publish_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "published", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def publish_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[TypeEvent]: + """Publish an event once name, location, dates, and a Luma embed are set.""" + event = _get_event_or_raise(event_id) + assert_ready_to_publish(event) + event = set_event_status(event, EventStatus.PUBLISHED) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishEventPermission], + trace_name="unpublish_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "unpublished", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def unpublish_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[TypeEvent]: + """Move an event back to draft.""" + event = _get_event_or_raise(event_id) + event = set_event_status(event, EventStatus.DRAFT) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[DeleteEventPermission], + trace_name="delete_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "deleted", + "get_data": lambda result, **kwargs: { + "event_id": str(kwargs.get("event_id")), + "success": result, + }, + }, + ) + def delete_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[bool]: + """Hard-delete an event.""" + event = _get_event_or_raise(event_id) + event.delete() + return MutationResponse.success_response(True) + + +def _update_kwargs(input: UpdateEventInput) -> dict: + kwargs: dict = {} + if input.name is not strawberry.UNSET: + kwargs["name"] = input.name + if input.location is not strawberry.UNSET: + kwargs["location"] = input.location + if input.start_date is not strawberry.UNSET: + kwargs["start_date"] = input.start_date + if input.end_date is not strawberry.UNSET: + kwargs["end_date"] = input.end_date + if input.luma_embed is not strawberry.UNSET: + kwargs["luma_embed"] = input.luma_embed + return kwargs + + +def _get_event_or_raise(event_id: uuid.UUID) -> Event: + try: + return Event.objects.get(id=event_id) + except Event.DoesNotExist: + raise DjangoValidationError(f"Event with id {event_id} does not exist.") diff --git a/api/schema/schema.py b/api/schema/schema.py index 5f919cfc..59584dfb 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -11,6 +11,7 @@ import api.schema.aimodel_schema import api.schema.collaborative_schema import api.schema.dataset_schema +import api.schema.event_schema import api.schema.dataspace_schema import api.schema.geography_schema import api.schema.metadata_schema @@ -78,6 +79,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.user_schema.Query, api.schema.collaborative_schema.Query, api.schema.publication_schema.Query, + api.schema.event_schema.Query, api.schema.platform_import_schema.Query, AuthQuery, ), @@ -102,6 +104,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.tags_schema.Mutation, api.schema.collaborative_schema.Mutation, api.schema.publication_schema.Mutation, + api.schema.event_schema.Mutation, api.schema.platform_import_schema.Mutation, AuthMutation, ), diff --git a/api/services/event_service.py b/api/services/event_service.py new file mode 100644 index 00000000..6a070cc7 --- /dev/null +++ b/api/services/event_service.py @@ -0,0 +1,185 @@ +""" +event_service +────────────── +Domain helpers for the Event CRUD flow. GraphQL resolvers stay thin and call +these with plain values. +""" + +import datetime +from typing import Any, List, Optional + +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db import transaction +from django.db.models import QuerySet + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.luma import canonical_luma_src, validate_luma_embed + +_UNSET: Any = object() + + +def create_event( + *, + user: Any, + organization: Any, + name: str = "", + location: str = "", + start_date: Any = None, + end_date: Any = None, + luma_embed: Optional[str] = None, +) -> Event: + """Create a draft. A blank name is replaced with a generated one. + + An organization on the request owns the row; otherwise the user does. + A sent embed must already be a Luma embed URL. Dates, when both are sent, + must run forward. + """ + errors: dict[str, List[str]] = {} + embed = _normalize_embed(luma_embed, errors) + _check_date_order(start_date, end_date, errors) + if errors: + raise DjangoValidationError(errors) + + return Event.objects.create( + name=(name or "").strip() or _default_name(), + location=(location or "").strip(), + start_date=start_date, + end_date=end_date, + luma_embed=embed or {}, + organization=organization, + user=user, + status=EventStatus.DRAFT, + ) + + +def apply_event_update( + event: Event, + *, + name: Any = _UNSET, + location: Any = _UNSET, + start_date: Any = _UNSET, + end_date: Any = _UNSET, + luma_embed: Any = _UNSET, +) -> Event: + """Apply a partial update. Omitted fields are left untouched. + + A draft may be saved incomplete. A row that is already published cannot be + saved back into an incomplete state. + """ + errors: dict[str, List[str]] = {} + new_name = event.name if name is _UNSET else (name or "").strip() + new_location = event.location if location is _UNSET else (location or "").strip() + new_start = event.start_date if start_date is _UNSET else start_date + new_end = event.end_date if end_date is _UNSET else end_date + + new_embed = event.luma_embed or {} + if luma_embed is not _UNSET: + if luma_embed: + parsed = _normalize_embed(luma_embed, errors) + if parsed is not None: + new_embed = parsed + else: + new_embed = {} + + _check_date_order(new_start, new_end, errors) + if errors: + raise DjangoValidationError(errors) + + if event.status == EventStatus.PUBLISHED: + _require_complete( + name=new_name, + location=new_location, + start_date=new_start, + end_date=new_end, + luma_embed=new_embed, + ) + + with transaction.atomic(): + event.name = new_name + event.location = new_location + event.start_date = new_start + event.end_date = new_end + event.luma_embed = new_embed + event.save() + return event + + +def assert_ready_to_publish(event: Event) -> None: + """Reject a publish when the draft is still missing a required field.""" + _require_complete( + name=event.name, + location=event.location, + start_date=event.start_date, + end_date=event.end_date, + luma_embed=event.luma_embed or {}, + ) + + +def set_event_status(event: Event, status: EventStatus) -> Event: + """Flip an event's publish status and save it.""" + event.status = status + event.save() + return event + + +def get_scoped_events(*, user: Any, organization: Any) -> "QuerySet[Event, Event]": + """Return the events a caller may list. + + Organization context → that org's events; an authenticated individual → + their own; anonymous → published only. Ordered newest-first. + """ + if organization: + queryset = Event.objects.filter(organization=organization) + elif getattr(user, "is_superuser", False): + queryset = Event.objects.all() + elif getattr(user, "is_authenticated", False): + queryset = Event.objects.filter(user=user, organization__isnull=True) + else: + queryset = Event.objects.filter(status=EventStatus.PUBLISHED) + + return queryset.select_related("organization", "user").order_by("-modified") + + +def _require_complete( + *, + name: Optional[str], + location: Optional[str], + start_date: Any, + end_date: Any, + luma_embed: Any, +) -> None: + errors: dict[str, List[str]] = {} + if not name or not name.strip(): + errors["name"] = ["Name is required."] + if not location or not location.strip(): + errors["location"] = ["Location is required."] + if start_date is None: + errors["start_date"] = ["Start date is required."] + if end_date is None: + errors["end_date"] = ["End date is required."] + _check_date_order(start_date, end_date, errors) + src = luma_embed.get("src") if isinstance(luma_embed, dict) else "" + if not src or not canonical_luma_src(str(src)): + errors["luma_embed"] = ["A Luma embed is required."] + if errors: + raise DjangoValidationError(errors) + + +def _normalize_embed(value: Optional[str], errors: dict[str, List[str]]) -> Optional[dict]: + if value is None or not str(value).strip(): + return None + try: + return validate_luma_embed(value) + except DjangoValidationError as exc: + errors["luma_embed"] = [str(message) for message in exc.messages] + return None + + +def _check_date_order(start_date: Any, end_date: Any, errors: dict[str, List[str]]) -> None: + if start_date and end_date and end_date < start_date: + errors["end_date"] = ["End date must be on or after the start date."] + + +def _default_name() -> str: + return f"New event {datetime.datetime.now().strftime('%d %b %Y - %H:%M:%S')}" diff --git a/api/types/type_event.py b/api/types/type_event.py new file mode 100644 index 00000000..78b9fb28 --- /dev/null +++ b/api/types/type_event.py @@ -0,0 +1,104 @@ +import uuid +from datetime import date, datetime +from typing import List, Optional + +import strawberry +import strawberry_django +from strawberry.enum import EnumType + +from api.models import Event +from api.types.base_type import BaseType +from api.types.type_organization import TypeOrganization +from api.utils.enums import EventStatus +from api.utils.qs_utils import StatusCount +from authorization.types import TypeUser + +event_status: EnumType = strawberry.enum(EventStatus) # type: ignore + + +def _text(value: object) -> Optional[str]: + return value if isinstance(value, str) and value else None + + +@strawberry.type +class TypeLumaEmbed: + """Presentation attributes taken from a Luma iframe. ``src`` is always a Luma embed URL.""" + + src: str + width: Optional[str] + height: Optional[str] + style: Optional[str] + title: Optional[str] + allow: Optional[str] + allowfullscreen: bool + frameborder: Optional[str] + referrerpolicy: Optional[str] + loading: Optional[str] + + +@strawberry_django.filter(Event) +class EventFilter: + id: Optional[uuid.UUID] + status: Optional[event_status] + + +@strawberry_django.order(Event) +class EventOrder: + name: strawberry.auto + created: strawberry.auto + modified: strawberry.auto + start_date: strawberry.auto + + +@strawberry_django.type(Event, filters=EventFilter, pagination=True, order=EventOrder) # type: ignore +class TypeEvent(BaseType): + """A Luma event in the dashboard provider flow.""" + + id: uuid.UUID + name: str + slug: str + location: str + start_date: Optional[date] + end_date: Optional[date] + status: event_status + created: datetime + modified: datetime + organization: Optional["TypeOrganization"] + user: Optional["TypeUser"] + + @strawberry.field + def luma_embed(self) -> Optional["TypeLumaEmbed"]: + """Whitelisted iframe attributes. Empty when the event has no embed yet.""" + raw = self.luma_embed if isinstance(getattr(self, "luma_embed", None), dict) else {} + src = raw.get("src") + if not isinstance(src, str) or not src: + return None + return TypeLumaEmbed( + src=src, + width=_text(raw.get("width")), + height=_text(raw.get("height")), + style=_text(raw.get("style")), + title=_text(raw.get("title")), + allow=_text(raw.get("allow")), + allowfullscreen=bool(raw.get("allowfullscreen")), + frameborder=_text(raw.get("frameborder")), + referrerpolicy=_text(raw.get("referrerpolicy")), + loading=_text(raw.get("loading")), + ) + + @strawberry.field + def is_individual_event(self) -> bool: + """True when owned by an individual rather than an organization.""" + return self.organization is None + + +@strawberry.type +class EventResponse: + """One page of ``eventsTable``: rows, the filtered total, and tab counts.""" + + data: List[TypeEvent] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter.", + ) diff --git a/api/utils/enums.py b/api/utils/enums.py index 49e1a610..5b8874de 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -197,6 +197,11 @@ class PublicationStatus(models.TextChoices): PUBLISHED = "PUBLISHED" +class EventStatus(models.TextChoices): + DRAFT = "DRAFT" + PUBLISHED = "PUBLISHED" + + class PublicationBlockType(models.TextChoices): FILE = "FILE" YOUTUBE = "YOUTUBE" diff --git a/api/utils/luma.py b/api/utils/luma.py new file mode 100644 index 00000000..d7415da2 --- /dev/null +++ b/api/utils/luma.py @@ -0,0 +1,175 @@ +""" +luma +──── +Parse a pasted Luma iframe into a small set of attributes. + +The stored value is not the raw HTML. ``src`` must be an ``https`` embed on +``lu.ma`` or ``luma.com``. Width, height, and the other presentation +attributes Luma includes are kept. Event handlers, ``srcdoc``, and any tag +other than one iframe are rejected. +""" + +import re +from html.parser import HTMLParser +from typing import Optional +from urllib.parse import urlparse, urlunparse + +from django.core.exceptions import ValidationError + +_LUMA_HOSTS = {"lu.ma", "www.lu.ma", "luma.com", "www.luma.com"} +_SIZE = re.compile(r"^\d{1,4}(px|%)?$") +_ALLOW = re.compile(r"^[A-Za-z0-9; _-]+$") +_UNSAFE_STYLE = re.compile( + r"javascript:|expression\s*\(|url\s*\(|@import|behavior\s*:|[<>\"']", + re.IGNORECASE, +) +_REFERRERS = { + "no-referrer", + "no-referrer-when-downgrade", + "origin", + "origin-when-cross-origin", + "same-origin", + "strict-origin", + "strict-origin-when-cross-origin", + "unsafe-url", +} +_LOADING = {"eager", "lazy"} + + +class _IframeParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.attrs: Optional[dict[str, str]] = None + self.iframe_count = 0 + self.other_tags = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + name = tag.lower() + if name == "iframe": + self.iframe_count += 1 + if self.iframe_count == 1: + self.attrs = {key.lower(): value or "" for key, value in attrs} + return + self.other_tags = True + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + self.handle_starttag(tag, attrs) + + +def canonical_luma_src(value: Optional[str]) -> Optional[str]: + """Return a canonical Luma embed URL, or None.""" + if not value or not value.strip(): + return None + parsed = urlparse(value.strip()) + if parsed.scheme != "https": + return None + if parsed.username or parsed.password: + return None + host = (parsed.hostname or "").lower() + if host not in _LUMA_HOSTS: + return None + if not parsed.path.startswith("/embed/"): + return None + return urlunparse(("https", host, parsed.path, "", parsed.query, "")) + + +def parse_luma_embed(value: Optional[str]) -> Optional[dict[str, object]]: + """Return whitelisted iframe attributes, or None when the paste is not a Luma embed.""" + if not value or not value.strip(): + return None + + trimmed = value.strip() + parser = _IframeParser() + parser.feed(trimmed) + parser.close() + if parser.other_tags or parser.iframe_count > 1: + return None + if parser.iframe_count == 1 and parser.attrs is not None: + return _attributes(parser.attrs) + if parser.iframe_count == 0: + src = canonical_luma_src(trimmed) + return {"src": src} if src else None + return None + + +def validate_luma_embed(value: Optional[str]) -> dict[str, object]: + """Return the whitelisted attributes, or raise ValidationError.""" + embed = parse_luma_embed(value) + if not embed: + raise ValidationError("Enter a valid Luma embed URL or iframe.") + return embed + + +def extract_luma_embed_url(value: Optional[str]) -> Optional[str]: + """Return only the embed URL. Kept for callers that do not need the other attributes.""" + embed = parse_luma_embed(value) + if not embed: + return None + src = embed.get("src") + return src if isinstance(src, str) else None + + +def _attributes(attrs: dict[str, str]) -> Optional[dict[str, object]]: + if any(name.startswith("on") or name in {"srcdoc", "formaction"} for name in attrs): + return None + src = canonical_luma_src(attrs.get("src")) + if not src: + return None + + embed: dict[str, object] = {"src": src} + width = _size(attrs.get("width")) + height = _size(attrs.get("height")) + style = _style(attrs.get("style")) + title = _plain(attrs.get("title"), 200) + allow = _allow(attrs.get("allow")) + frameborder = _size(attrs.get("frameborder")) + referrer = (attrs.get("referrerpolicy") or "").strip().lower() + loading = (attrs.get("loading") or "").strip().lower() + + if width: + embed["width"] = width + if height: + embed["height"] = height + if style: + embed["style"] = style + if title: + embed["title"] = title + if allow: + embed["allow"] = allow + if "allowfullscreen" in attrs: + embed["allowfullscreen"] = True + if frameborder: + embed["frameborder"] = frameborder + if referrer in _REFERRERS: + embed["referrerpolicy"] = referrer + if loading in _LOADING: + embed["loading"] = loading + return embed + + +def _size(value: Optional[str]) -> Optional[str]: + cleaned = (value or "").strip() + if _SIZE.match(cleaned): + return cleaned + return None + + +def _style(value: Optional[str]) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or len(cleaned) > 500 or _UNSAFE_STYLE.search(cleaned): + return None + return cleaned + + +def _plain(value: Optional[str], limit: int) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or "<" in cleaned or ">" in cleaned: + return None + return cleaned[:limit] + + +def _allow(value: Optional[str]) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or len(cleaned) > 200 or not _ALLOW.match(cleaned): + return None + return cleaned diff --git a/authorization/permissions.py b/authorization/permissions.py index ce818e9b..fb27f87f 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -5,8 +5,8 @@ from strawberry.permission import BasePermission from strawberry.types import Info -from api.models import Dataset, Organization, Publication -from api.utils.enums import PublicationStatus +from api.models import Dataset, Event, Organization, Publication +from api.utils.enums import EventStatus, PublicationStatus from authorization.models import DatasetPermission, OrganizationMembership, Role # Roles that may publish/unpublish and edit an org-owned publication, by name. @@ -622,3 +622,137 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: if source.status == PublicationStatus.PUBLISHED.value: return True return bool(getattr(request, "user", None) and request.user.is_authenticated) + + +# --------------------------------------------------------------------------- +# Event permissions — same ownership rules as publications. +# --------------------------------------------------------------------------- + + +def _resolve_event_id(kwargs: Any) -> Optional[Any]: + """Pull the target event id out of a mutation's arguments.""" + event_id = kwargs.get("event_id") + if event_id: + return event_id + payload = kwargs.get("input") + if payload is not None and getattr(payload, "id", None): + return payload.id + return None + + +def _user_manages_event(user: Any, event: Event, operation: str) -> bool: + """Whether a user may perform ``operation`` on an event.""" + if user.is_superuser: + return True + if event.user and event.user == user: + return True + if not event.organization: + return False + + membership = OrganizationMembership.objects.filter( + user=user, organization=event.organization + ).first() + if not membership: + return False + + role = membership.role + if operation == "publish": + return role.name in PUBLICATION_MANAGER_ROLE_NAMES + if operation == "delete": + return role.can_delete + return role.can_change + + +class EventPermissionGraphQL(BasePermission): # type: ignore[misc] + """Base event mutation permission — keys on the event id in kwargs.""" + + message = "You don't have permission to modify this event" + operation = "change" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + event_id = _resolve_event_id(kwargs) + if not event_id: + return False + + try: + event = Event.objects.get(id=event_id) + except Event.DoesNotExist: + return False + + return _user_manages_event(user, event, self.operation) + + +class ChangeEventPermission(EventPermissionGraphQL): + operation = "change" + + +class DeleteEventPermission(EventPermissionGraphQL): + message = "You don't have permission to delete this event" + operation = "delete" + + +class PublishEventPermission(EventPermissionGraphQL): + message = "You don't have permission to publish this event" + operation = "publish" + + +class CreateEventPermission(BasePermission): # type: ignore[misc] + """Any authenticated user may create an individual event. + + Creating inside an organization requires the ``add`` role. + """ + + message = "You don't have permission to create an event" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + organization = info.context.context.get("organization") + if organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + return bool(membership and membership.role.can_add) + + return True + + +class AllowPublishedEvents(BasePermission): # type: ignore[misc] + """A published event is world-readable. A draft is visible to its owner.""" + + message = "You need to be authenticated to access non-published events" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + request = info.context + event_id = kwargs.get("event_id") + + if event_id: + try: + event = Event.objects.get(id=event_id) + except Event.DoesNotExist: + return True + + if event.status == EventStatus.PUBLISHED.value: + return True + + user = request.user + if not user.is_authenticated: + return False + if user.is_superuser: + return True + if event.user and event.user == user: + return True + if event.organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=event.organization + ).first() + return bool(membership and membership.role.can_view) + return False + + return bool(getattr(request, "user", None) and request.user.is_authenticated) diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py new file mode 100644 index 00000000..521e4e7a --- /dev/null +++ b/tests/schema/test_event_schema.py @@ -0,0 +1,393 @@ +"""GraphQL tests for the event provider.""" + +import types +from datetime import date +from unittest.mock import patch + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Event +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import EventStatus +from api.utils.luma import extract_luma_embed_url, parse_luma_embed +from authorization.models import OrganizationMembership, Role, User + +EMBED = "https://lu.ma/embed/event/evt-abc/simple" +IFRAME = ( + f'' +) + + +@pytest.fixture(autouse=True) +def _no_activity_recording(): + with patch("api.schema.base_mutation.record_activity", return_value=None): + yield + + +@pytest.fixture +def roles(db): + admin = Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ) + auditor = Role.objects.create( + name="auditor", can_view=True, can_add=False, can_change=False, can_delete=False + ) + return {"admin": admin, "auditor": auditor} + + +@pytest.fixture +def org_a(db): + return Organization.objects.create(name="Org A", description="a", slug="org-a") + + +@pytest.fixture +def org_b(db): + return Organization.objects.create(name="Org B", description="b", slug="org-b") + + +def _member(user, org, role): + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +@pytest.fixture +def org_a_admin(roles, org_a): + return _member( + User.objects.create(username="event_a_admin", keycloak_id="event_a_admin"), + org_a, + roles["admin"], + ) + + +@pytest.fixture +def org_a_auditor(roles, org_a): + return _member( + User.objects.create(username="event_a_auditor", keycloak_id="event_a_auditor"), + org_a, + roles["auditor"], + ) + + +@pytest.fixture +def org_b_admin(roles, org_b): + return _member( + User.objects.create(username="event_b_admin", keycloak_id="event_b_admin"), + org_b, + roles["admin"], + ) + + +@pytest.fixture +def individual(db): + return User.objects.create(username="event_solo", keycloak_id="event_solo") + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, + context={"organization": organization} if organization else {}, + ) + + +def run(query, context, variables=None): + return schema.execute_sync(query, variable_values=variables or {}, context_value=context) + + +def _make_event(user, org, **overrides): + values = { + "name": "Existing meetup", + "location": "Pune", + "start_date": date(2026, 6, 1), + "end_date": date(2026, 6, 2), + "luma_embed": {"src": EMBED}, + "organization": org, + "user": user, + "status": EventStatus.DRAFT, + } + values.update(overrides) + return Event.objects.create(**values) + + +CREATE = """ +mutation Create($input: CreateEventInput!) { + createEvent(input: $input) { + success + errors { nonFieldErrors fieldErrors { field messages } } + data { id status isIndividualEvent organization { id } user { id } } + } +} +""" + +UPDATE = """ +mutation Update($input: UpdateEventInput!) { + updateEvent(input: $input) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id name location startDate endDate lumaEmbed { src width height style allowfullscreen } } + } +} +""" + +PUBLISH = """ +mutation Publish($id: UUID!) { + publishEvent(eventId: $id) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id status } + } +} +""" + +TABLE = """ +query Events($filters: [FilterSpec!]) { + eventsTable(filters: $filters) { + totalItemsCount + data { id status name } + } +} +""" + +GET = """ +query Get($id: UUID!) { + event(eventId: $id) { id status } +} +""" + +PUBLISHED = """ +query Published($organizationSlug: String) { + publishedEvents(organizationSlug: $organizationSlug) { + totalItemsCount + data { name slug status organization { slug } } + } +} +""" + +BY_SLUG = """ +query BySlug($slug: String!) { + eventBySlug(slug: $slug) { name slug status lumaEmbed { src } } +} +""" + + +def test_iframe_keeps_presentation_attributes(): + embed = parse_luma_embed(IFRAME) + assert embed is not None + assert embed["src"] == EMBED + assert embed["width"] == "600" + assert embed["height"] == "450" + assert embed["allowfullscreen"] is True + assert "border-radius" in str(embed["style"]) + + +def test_iframe_src_becomes_a_canonical_embed_url(): + assert extract_luma_embed_url(IFRAME) == EMBED + + +def test_script_and_event_handlers_are_rejected(): + assert parse_luma_embed(f'') is None + assert parse_luma_embed(f'') is None + + +def test_non_embed_luma_page_is_rejected(): + assert extract_luma_embed_url("https://lu.ma/community-meetup") is None + assert extract_luma_embed_url("http://lu.ma/embed/event/evt-abc/simple") is None + assert extract_luma_embed_url("https://evil.example/embed/event/evt-abc") is None + + +@pytest.mark.django_db +class TestEventSchema: + def test_org_member_creates_an_org_draft(self, org_a_admin, org_a): + result = run(CREATE, ctx(org_a_admin, org_a), {"input": {}}) + + assert result.errors is None + payload = result.data["createEvent"] + assert payload["success"] is True + assert payload["data"]["status"] == "DRAFT" + assert payload["data"]["organization"]["id"] == str(org_a.id) + assert payload["data"]["isIndividualEvent"] is False + + def test_individual_creates_a_user_owned_draft(self, individual): + result = run(CREATE, ctx(individual), {"input": {"name": "Solo night"}}) + + payload = result.data["createEvent"] + assert payload["success"] is True + assert payload["data"]["isIndividualEvent"] is True + assert payload["data"]["user"]["id"] == str(individual.id) + + def test_auditor_cannot_create(self, org_a_auditor, org_a): + result = run(CREATE, ctx(org_a_auditor, org_a), {"input": {}}) + + assert result.data["createEvent"]["success"] is False + assert Event.objects.count() == 0 + + def test_anonymous_cannot_create(self): + result = run(CREATE, ctx(AnonymousUser()), {"input": {}}) + + assert result.data["createEvent"]["success"] is False + + def test_invalid_embed_is_rejected(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + {"input": {"id": event_id, "lumaEmbed": "https://youtu.be/dQw4w9WgXcQ"}}, + ) + + assert result.data["updateEvent"]["success"] is False + event = Event.objects.get(id=event_id) + assert event.luma_embed == {} + + def test_iframe_is_stored_as_the_embed_url(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + { + "input": { + "id": event_id, + "name": "Community meetup", + "location": "Pune", + "startDate": "2026-06-01", + "endDate": "2026-06-02", + "lumaEmbed": IFRAME, + } + }, + ) + + payload = result.data["updateEvent"] + assert result.errors is None + assert payload["success"] is True + assert payload["data"]["lumaEmbed"]["src"] == EMBED + assert payload["data"]["lumaEmbed"]["width"] == "600" + assert payload["data"]["lumaEmbed"]["height"] == "450" + assert payload["data"]["lumaEmbed"]["allowfullscreen"] is True + + def test_end_date_before_start_is_rejected(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + {"input": {"id": event_id, "startDate": "2026-06-02", "endDate": "2026-06-01"}}, + ) + + assert result.data["updateEvent"]["success"] is False + + def test_publish_rejects_an_incomplete_draft(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, location="", luma_embed={}) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(event.id)}) + + assert result.data["publishEvent"]["success"] is False + event.refresh_from_db() + assert event.status == EventStatus.DRAFT + + def test_publish_accepts_a_complete_event(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(event.id)}) + + assert result.errors is None + assert result.data["publishEvent"]["success"] is True + assert result.data["publishEvent"]["data"]["status"] == "PUBLISHED" + + def test_listing_is_org_scoped(self, org_a_admin, org_b_admin, org_a, org_b): + _make_event(org_a_admin, org_a) + _make_event(org_b_admin, org_b, name="Other org") + + result = run(TABLE, ctx(org_a_admin, org_a)) + + assert result.errors is None + names = [row["name"] for row in result.data["eventsTable"]["data"]] + assert names == ["Existing meetup"] + + def test_anonymous_listing_is_published_only(self, org_a_admin, org_a): + _make_event(org_a_admin, org_a, name="Draft night") + _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + + result = run(TABLE, ctx(AnonymousUser())) + + statuses = [row["status"] for row in result.data["eventsTable"]["data"]] + assert statuses == ["PUBLISHED"] + + def test_published_listing_is_public_for_a_signed_in_user( + self, org_a_admin, org_b_admin, org_a, org_b, individual + ): + _make_event(org_a_admin, org_a, name="Draft night") + _make_event( + org_a_admin, + org_a, + name="Public night", + status=EventStatus.PUBLISHED, + start_date=date(2026, 6, 2), + ) + _make_event( + org_b_admin, + org_b, + name="Other public", + status=EventStatus.PUBLISHED, + start_date=date(2026, 6, 1), + ) + + result = run(PUBLISHED, ctx(individual)) + + assert result.errors is None + names = [row["name"] for row in result.data["publishedEvents"]["data"]] + assert names == ["Other public", "Public night"] + assert result.data["publishedEvents"]["totalItemsCount"] == 2 + + def test_published_listing_can_be_limited_to_one_organization( + self, org_a_admin, org_b_admin, org_a, org_b + ): + _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + _make_event(org_b_admin, org_b, name="Other public", status=EventStatus.PUBLISHED) + + result = run(PUBLISHED, ctx(AnonymousUser()), {"organizationSlug": "org-a"}) + + names = [row["name"] for row in result.data["publishedEvents"]["data"]] + assert names == ["Public night"] + + def test_event_by_slug_returns_a_published_event(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + + result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.slug}) + + assert result.errors is None + assert result.data["eventBySlug"]["slug"] == event.slug + assert result.data["eventBySlug"]["lumaEmbed"]["src"] == EMBED + + def test_event_by_slug_hides_drafts(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, name="Draft night") + + result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.slug}) + + assert result.errors is None + assert result.data["eventBySlug"] is None + + def test_anonymous_cannot_read_a_draft(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a) + + result = run(GET, ctx(AnonymousUser()), {"id": str(event.id)}) + + assert result.errors is not None + + def test_other_org_cannot_update(self, org_a_admin, org_b_admin, org_a, org_b): + event = _make_event(org_a_admin, org_a) + + result = run( + UPDATE, + ctx(org_b_admin, org_b), + {"input": {"id": str(event.id), "name": "Taken"}}, + ) + + assert result.data["updateEvent"]["success"] is False + event.refresh_from_db() + assert event.name == "Existing meetup" From f1fe2347822e130033065b446d7553c1d4be500c Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 29 Oct 2026 01:53:51 +0530 Subject: [PATCH 22/55] fix(api): match isIndividualEvent to the Event model rule An event whose owner was deleted (user set to NULL) and has no organization was reported as individual by the API but not by Event.is_individual_event. Use the same check in both. --- api/types/type_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/types/type_event.py b/api/types/type_event.py index 78b9fb28..cf06c897 100644 --- a/api/types/type_event.py +++ b/api/types/type_event.py @@ -89,7 +89,7 @@ def luma_embed(self) -> Optional["TypeLumaEmbed"]: @strawberry.field def is_individual_event(self) -> bool: """True when owned by an individual rather than an organization.""" - return self.organization is None + return self.organization is None and self.user is not None @strawberry.type From b2fae581a307439e412ff9da7e8511601e44fca3 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 29 Oct 2026 11:48:13 +0530 Subject: [PATCH 23/55] feat(api): permanent public links for events Each event gets a 12-character public_id; its link is the name words plus that id (urlSlug), so renames never break links and two events can share a name. eventBySlug finds the event by the id at the end of the link. The slug is no longer unique and the counter loop is gone. Also drops the 0055_event dependency on 0049_merge_publication_and_accessmodel, which does not exist. --- api/migrations/0055_event.py | 1 - api/migrations/0056_event_public_id.py | 25 ++++ api/migrations/0057_fill_event_public_id.py | 17 +++ api/migrations/0058_event_public_id_unique.py | 15 +++ api/models/Event.py | 20 +-- api/schema/event_schema.py | 13 +- api/types/type_event.py | 8 +- api/utils/public_links.py | 118 ++++++++++++++++++ tests/schema/test_event_schema.py | 8 +- 9 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 api/migrations/0056_event_public_id.py create mode 100644 api/migrations/0057_fill_event_public_id.py create mode 100644 api/migrations/0058_event_public_id_unique.py create mode 100644 api/utils/public_links.py diff --git a/api/migrations/0055_event.py b/api/migrations/0055_event.py index 61b8d1db..ef4e2f3b 100644 --- a/api/migrations/0055_event.py +++ b/api/migrations/0055_event.py @@ -8,7 +8,6 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0049_merge_publication_and_accessmodel"), ("api", "0054_add_source_identifier_metadata"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] diff --git a/api/migrations/0056_event_public_id.py b/api/migrations/0056_event_public_id.py new file mode 100644 index 00000000..cf8c452a --- /dev/null +++ b/api/migrations/0056_event_public_id.py @@ -0,0 +1,25 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Add Event.public_id (nullable for now) and drop the unique rule on slug. + + Filled for existing rows in 0057, made unique and required in 0058. + """ + + dependencies = [ + ("api", "0055_event"), + ] + + operations = [ + migrations.AddField( + model_name="event", + name="public_id", + field=models.CharField(editable=False, max_length=12, null=True), + ), + migrations.AlterField( + model_name="event", + name="slug", + field=models.SlugField(max_length=255), + ), + ] diff --git a/api/migrations/0057_fill_event_public_id.py b/api/migrations/0057_fill_event_public_id.py new file mode 100644 index 00000000..7cffa33e --- /dev/null +++ b/api/migrations/0057_fill_event_public_id.py @@ -0,0 +1,17 @@ +from django.db import migrations + +from api.utils.public_links import fill_missing_public_ids + + +def fill(apps, schema_editor): + fill_missing_public_ids(apps.get_model("api", "Event")) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0056_event_public_id"), + ] + + operations = [ + migrations.RunPython(fill, migrations.RunPython.noop), + ] diff --git a/api/migrations/0058_event_public_id_unique.py b/api/migrations/0058_event_public_id_unique.py new file mode 100644 index 00000000..cab16b1e --- /dev/null +++ b/api/migrations/0058_event_public_id_unique.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0057_fill_event_public_id"), + ] + + operations = [ + migrations.AlterField( + model_name="event", + name="public_id", + field=models.CharField(editable=False, max_length=12, unique=True), + ), + ] diff --git a/api/models/Event.py b/api/models/Event.py index d51c358e..404b0cf0 100644 --- a/api/models/Event.py +++ b/api/models/Event.py @@ -5,6 +5,7 @@ from django.utils.text import slugify from api.utils.enums import EventStatus +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id class Event(models.Model): @@ -16,7 +17,9 @@ class Event(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=300, blank=True) - slug = models.SlugField(max_length=255, unique=True) + # Readable words only; not unique. Links and lookups use public_id. + slug = models.SlugField(max_length=255) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) location = models.CharField(max_length=300, blank=True) start_date = models.DateField(null=True, blank=True) end_date = models.DateField(null=True, blank=True) @@ -47,14 +50,13 @@ class Event(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - base_slug = slugify(self.name) or "untitled" - slug = base_slug - counter = 1 - while Event.objects.filter(slug=slug).exclude(pk=self.pk).exists(): - slug = f"{base_slug}-{counter}" - counter += 1 - self.slug = slug - super().save(*args, **kwargs) + self.slug = slugify(self.name)[:255] or "untitled" + save_with_public_id(self, lambda: super(Event, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + """Public link part: title words plus public_id, e.g. ``data-meetup-k3x9m2qa7b1z``.""" + return build_url_slug(self.name, self.public_id) @property def is_individual_event(self) -> bool: diff --git a/api/schema/event_schema.py b/api/schema/event_schema.py index 5ca45383..77d84b57 100644 --- a/api/schema/event_schema.py +++ b/api/schema/event_schema.py @@ -25,6 +25,7 @@ from api.types.type_event import EventResponse, TypeEvent from api.utils.enums import EventStatus from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, @@ -166,9 +167,17 @@ def published_events( @strawberry.field @trace_resolver(name="event_by_slug", attributes={"component": "event"}) def event_by_slug(self, info: Info, slug: str) -> Optional[TypeEvent]: - """Published event for a public page. Drafts and unknown slugs return null.""" + """Published event for a public page, found by its link (``urlSlug``). + + Only the publicId at the end of the link is used, so a link with old + title words still works. Drafts, unknown ids and links without an id + return null. + """ + public_id = parse_public_id(slug) + if public_id is None: + return None event = ( - Event.objects.filter(slug=slug, status=EventStatus.PUBLISHED) + Event.objects.filter(public_id=public_id, status=EventStatus.PUBLISHED) .select_related("organization", "user") .first() ) diff --git a/api/types/type_event.py b/api/types/type_event.py index cf06c897..c97e11c2 100644 --- a/api/types/type_event.py +++ b/api/types/type_event.py @@ -1,6 +1,6 @@ import uuid from datetime import date, datetime -from typing import List, Optional +from typing import List, Optional, cast import strawberry import strawberry_django @@ -57,6 +57,7 @@ class TypeEvent(BaseType): id: uuid.UUID name: str slug: str + public_id: str location: str start_date: Optional[date] end_date: Optional[date] @@ -86,6 +87,11 @@ def luma_embed(self) -> Optional["TypeLumaEmbed"]: loading=_text(raw.get("loading")), ) + @strawberry.field + def url_slug(self) -> str: + """Public link part: title words plus publicId. Use it in event page URLs.""" + return cast(Event, self).url_slug + @strawberry.field def is_individual_event(self) -> bool: """True when owned by an individual rather than an organization.""" diff --git a/api/utils/public_links.py b/api/utils/public_links.py new file mode 100644 index 00000000..03d05331 --- /dev/null +++ b/api/utils/public_links.py @@ -0,0 +1,118 @@ +"""Permanent public links: a short random ``public_id`` plus readable title words. + +A link looks like ``city-air-quality-index-k3x9m2qa7b1z``. Only the last part, +the ``public_id``, finds the record; the words come from the current title and +are there for people to read. Renaming a record changes the words, never the +id, so old links keep working. + +``public_id`` is unique per table, enforced by the database unique index. It is +never checked before saving (a check and an insert can race); a clash makes the +insert fail, and the save is retried with a new id. +""" + +from __future__ import annotations + +import re +import secrets +import string +import uuid +from typing import Callable, Optional + +from django.db import IntegrityError, models, transaction +from django.utils.text import slugify + +PUBLIC_ID_LENGTH = 12 +PUBLIC_ID_ALPHABET = string.ascii_lowercase + string.digits +URL_SLUG_WORDS_MAX = 50 + +_PUBLIC_ID_RE = re.compile(rf"[a-z0-9]{{{PUBLIC_ID_LENGTH}}}") + + +def generate_public_id() -> str: + """12 random characters from a-z and 0-9 (36^12, about 4.7 x 10^18 values).""" + return "".join(secrets.choice(PUBLIC_ID_ALPHABET) for _ in range(PUBLIC_ID_LENGTH)) + + +def save_with_public_id( + instance: models.Model, + save_fn: Callable[[], None], + field: str = "public_id", + max_attempts: int = 5, +) -> None: + """Save ``instance``, giving it a new ``public_id`` first if it has none. + + If the insert fails because another row already has that id, a new id is + generated and the save runs again. Any other database error is raised + unchanged. Each attempt runs in its own savepoint, so a failed attempt does + not break the caller's transaction. + """ + if getattr(instance, field): + save_fn() + return + + for attempt in range(1, max_attempts + 1): + setattr(instance, field, generate_public_id()) + try: + with transaction.atomic(): + save_fn() + return + except IntegrityError as exc: + if field not in str(exc) or attempt == max_attempts: + raise + + +def build_url_slug(title: Optional[str], public_id: str) -> str: + """Link part for a record: title words (max 50 chars, cut between words) + id. + + Returns only the id when the title has no Latin letters or digits (for + example a title written only in Hindi), since those are dropped by slugify. + """ + words = slugify(title or "") + if len(words) > URL_SLUG_WORDS_MAX: + cut = words[: URL_SLUG_WORDS_MAX + 1].rsplit("-", 1)[0] + # A single word longer than the limit has no boundary to cut at. + words = cut if len(cut) <= URL_SLUG_WORDS_MAX else words[:URL_SLUG_WORDS_MAX] + words = words.strip("-") + return f"{words}-{public_id}" if words else public_id + + +def parse_public_id(link_part: Optional[str]) -> Optional[str]: + """The ``public_id`` at the end of a link part, or None if there is none. + + ``city-air-quality-index-k3x9m2qa7b1z`` and ``k3x9m2qa7b1z`` both give + ``k3x9m2qa7b1z``. A UUID or a number gives None. An old slug whose last word + happens to be 12 letters long does parse, so callers should fall back to + their old lookup both when this returns None and when no record has the id. + """ + if not link_part: + return None + link_part = link_part.strip().strip("/") + # A UUID's last group is also 12 hex characters; it is an old link, not an id. + try: + uuid.UUID(link_part) + return None + except ValueError: + pass + candidate = link_part.rsplit("-", 1)[-1].lower() + return candidate if _PUBLIC_ID_RE.fullmatch(candidate) else None + + +def fill_missing_public_ids(model: type[models.Model], field: str = "public_id") -> int: + """Give every row of ``model`` that has no ``public_id`` a new one. + + For data migrations that add the column to an existing table, run before the + unique index is added. Uniqueness is kept here with a set of ids already in + the table, since the index that would catch a clash does not exist yet. + Returns the number of rows filled. + """ + manager = model._default_manager # type: ignore[attr-defined] + used = set(manager.exclude(**{f"{field}__isnull": True}).values_list(field, flat=True)) + filled = 0 + for pk in list(manager.filter(**{f"{field}__isnull": True}).values_list("pk", flat=True)): + value = generate_public_id() + while value in used: + value = generate_public_id() + used.add(value) + manager.filter(pk=pk).update(**{field: value}) + filled += 1 + return filled diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py index 521e4e7a..9672d5f0 100644 --- a/tests/schema/test_event_schema.py +++ b/tests/schema/test_event_schema.py @@ -167,7 +167,7 @@ def _make_event(user, org, **overrides): BY_SLUG = """ query BySlug($slug: String!) { - eventBySlug(slug: $slug) { name slug status lumaEmbed { src } } + eventBySlug(slug: $slug) { name urlSlug status lumaEmbed { src } } } """ @@ -358,16 +358,16 @@ def test_published_listing_can_be_limited_to_one_organization( def test_event_by_slug_returns_a_published_event(self, org_a_admin, org_a): event = _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) - result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.slug}) + result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.url_slug}) assert result.errors is None - assert result.data["eventBySlug"]["slug"] == event.slug + assert result.data["eventBySlug"]["urlSlug"] == event.url_slug assert result.data["eventBySlug"]["lumaEmbed"]["src"] == EMBED def test_event_by_slug_hides_drafts(self, org_a_admin, org_a): event = _make_event(org_a_admin, org_a, name="Draft night") - result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.slug}) + result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.url_slug}) assert result.errors is None assert result.data["eventBySlug"] is None From 5a8eb323ab38802484af5f1a6e72137bd68e3cd7 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 29 Oct 2026 11:57:50 +0530 Subject: [PATCH 24/55] fix(api): drop missing 0049 dependency from 0055_event --- api/migrations/0055_event.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/migrations/0055_event.py b/api/migrations/0055_event.py index 61b8d1db..ef4e2f3b 100644 --- a/api/migrations/0055_event.py +++ b/api/migrations/0055_event.py @@ -8,7 +8,6 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0049_merge_publication_and_accessmodel"), ("api", "0054_add_source_identifier_metadata"), migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] From 7bee5e5f36886b5df8e35be394336a2337dae67a Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 29 Oct 2026 14:29:31 +0530 Subject: [PATCH 25/55] refactor(api): fold event public_id into 0055 and drop Event.slug The event table is new in this change set, so public_id can be created with the table instead of added, filled and made unique in three later migrations. Event.slug was no longer read anywhere (links use urlSlug and lookups use public_id), so it is removed rather than kept as a dead column. --- api/migrations/0055_event.py | 2 +- api/migrations/0056_event_public_id.py | 25 ------------------- api/migrations/0057_fill_event_public_id.py | 17 ------------- api/migrations/0058_event_public_id_unique.py | 15 ----------- api/models/Event.py | 10 +++----- api/types/type_event.py | 1 - tests/schema/test_event_schema.py | 2 +- 7 files changed, 5 insertions(+), 67 deletions(-) delete mode 100644 api/migrations/0056_event_public_id.py delete mode 100644 api/migrations/0057_fill_event_public_id.py delete mode 100644 api/migrations/0058_event_public_id_unique.py diff --git a/api/migrations/0055_event.py b/api/migrations/0055_event.py index ef4e2f3b..fee78c6b 100644 --- a/api/migrations/0055_event.py +++ b/api/migrations/0055_event.py @@ -23,7 +23,7 @@ class Migration(migrations.Migration): ), ), ("name", models.CharField(blank=True, max_length=300)), - ("slug", models.SlugField(max_length=255, unique=True)), + ("public_id", models.CharField(editable=False, max_length=12, unique=True)), ("location", models.CharField(blank=True, max_length=300)), ("start_date", models.DateField(blank=True, null=True)), ("end_date", models.DateField(blank=True, null=True)), diff --git a/api/migrations/0056_event_public_id.py b/api/migrations/0056_event_public_id.py deleted file mode 100644 index cf8c452a..00000000 --- a/api/migrations/0056_event_public_id.py +++ /dev/null @@ -1,25 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - """Add Event.public_id (nullable for now) and drop the unique rule on slug. - - Filled for existing rows in 0057, made unique and required in 0058. - """ - - dependencies = [ - ("api", "0055_event"), - ] - - operations = [ - migrations.AddField( - model_name="event", - name="public_id", - field=models.CharField(editable=False, max_length=12, null=True), - ), - migrations.AlterField( - model_name="event", - name="slug", - field=models.SlugField(max_length=255), - ), - ] diff --git a/api/migrations/0057_fill_event_public_id.py b/api/migrations/0057_fill_event_public_id.py deleted file mode 100644 index 7cffa33e..00000000 --- a/api/migrations/0057_fill_event_public_id.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.db import migrations - -from api.utils.public_links import fill_missing_public_ids - - -def fill(apps, schema_editor): - fill_missing_public_ids(apps.get_model("api", "Event")) - - -class Migration(migrations.Migration): - dependencies = [ - ("api", "0056_event_public_id"), - ] - - operations = [ - migrations.RunPython(fill, migrations.RunPython.noop), - ] diff --git a/api/migrations/0058_event_public_id_unique.py b/api/migrations/0058_event_public_id_unique.py deleted file mode 100644 index cab16b1e..00000000 --- a/api/migrations/0058_event_public_id_unique.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("api", "0057_fill_event_public_id"), - ] - - operations = [ - migrations.AlterField( - model_name="event", - name="public_id", - field=models.CharField(editable=False, max_length=12, unique=True), - ), - ] diff --git a/api/models/Event.py b/api/models/Event.py index 404b0cf0..0b6408fd 100644 --- a/api/models/Event.py +++ b/api/models/Event.py @@ -2,7 +2,6 @@ from typing import Any from django.db import models -from django.utils.text import slugify from api.utils.enums import EventStatus from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id @@ -11,14 +10,13 @@ class Event(models.Model): """A Luma-backed event owned by an organization or an individual user. - Peer to Publication: a UUID/slug entity with a draft/publish status. The - embed is stored as whitelisted iframe attributes, never as raw HTML. + Peer to Publication: a UUID entity with a draft/publish status, linked by + ``public_id``. The embed is stored as whitelisted iframe attributes, never + as raw HTML. """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=300, blank=True) - # Readable words only; not unique. Links and lookups use public_id. - slug = models.SlugField(max_length=255) public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) location = models.CharField(max_length=300, blank=True) start_date = models.DateField(null=True, blank=True) @@ -49,8 +47,6 @@ class Event(models.Model): modified = models.DateTimeField(auto_now=True) def save(self, *args: Any, **kwargs: Any) -> None: - if not self.slug: - self.slug = slugify(self.name)[:255] or "untitled" save_with_public_id(self, lambda: super(Event, self).save(*args, **kwargs)) @property diff --git a/api/types/type_event.py b/api/types/type_event.py index c97e11c2..0df3c110 100644 --- a/api/types/type_event.py +++ b/api/types/type_event.py @@ -56,7 +56,6 @@ class TypeEvent(BaseType): id: uuid.UUID name: str - slug: str public_id: str location: str start_date: Optional[date] diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py index 9672d5f0..83edbc57 100644 --- a/tests/schema/test_event_schema.py +++ b/tests/schema/test_event_schema.py @@ -160,7 +160,7 @@ def _make_event(user, org, **overrides): query Published($organizationSlug: String) { publishedEvents(organizationSlug: $organizationSlug) { totalItemsCount - data { name slug status organization { slug } } + data { name urlSlug status organization { slug } } } } """ From 9f1624884aed4f11345e68b3bb57f969d7c41f51 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Thu, 29 Oct 2026 21:42:52 +0530 Subject: [PATCH 26/55] fix(api): add permission checks to dataset resources, charts and access models Mutations on resources, charts, chart images and access models, and updatePromptResource, now need edit rights on the dataset they belong to, using the same rules as dataset edits. Queries for these objects return draft datasets' items only to people who can see the draft. Shared rules live in api/utils/dataset_access.py. --- api/schema/access_model_schema.py | 48 ++++++++-- api/schema/dataset_schema.py | 18 +++- api/schema/resource_chart_schema.py | 74 +++++++++++++-- api/schema/resource_schema.py | 60 +++++++++++-- api/schema/resoure_chart_image_schema.py | 89 ++++++++++++------ api/schema/schema.py | 5 +- api/types/type_resource_chart_image.py | 10 ++- api/utils/dataset_access.py | 109 +++++++++++++++++++++++ 8 files changed, 364 insertions(+), 49 deletions(-) create mode 100644 api/utils/dataset_access.py diff --git a/api/schema/access_model_schema.py b/api/schema/access_model_schema.py index d5dece23..6c320327 100644 --- a/api/schema/access_model_schema.py +++ b/api/schema/access_model_schema.py @@ -18,6 +18,11 @@ ResourceSchema, ) from api.types.type_access_model import TypeAccessModel +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import AccessTypes AccessTypesEnum = strawberry.enum(AccessTypes) # type: ignore @@ -52,15 +57,37 @@ class EditAccessModelInput: class Query: @strawberry_django.field def access_model_resources(self, info: Info, dataset_id: uuid.UUID) -> List[TypeAccessModel]: - models = AccessModel.objects.filter(dataset_id=dataset_id) + models = AccessModel.objects.filter(dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeAccessModel.from_django(model) for model in models] @strawberry_django.field def access_model(self, info: Info, access_model_id: uuid.UUID) -> TypeAccessModel: - model = AccessModel.objects.get(id=access_model_id) + model = AccessModel.objects.select_related("dataset").get(id=access_model_id) + if not can_view_dataset(info.context.user, model.dataset): + # Same answer as a missing ID. + raise AccessModel.DoesNotExist("AccessModel matching query does not exist.") return TypeAccessModel.from_django(model) +class ChangeAccessModelPermission(ChangeDatasetPartPermission): + """Access models belong to a dataset; changing one needs edit rights on it.""" + + message = "You don't have permission to change this dataset's access models" + + def datasets(self, kwargs: Any) -> Optional[list]: + data = kwargs.get("access_model_input") + access_model_id = kwargs.get("access_model_id") or getattr(data, "access_model_id", None) + if access_model_id: + # An existing access model stays in its own dataset whatever the input says. + model = AccessModel.objects.select_related("dataset").filter(id=access_model_id).first() + return [model.dataset] if model else None + dataset_id = getattr(data, "dataset", None) + dataset = Dataset.objects.filter(id=dataset_id).first() if dataset_id else None + return [dataset] if dataset else None + + def _add_resource_fields( access_model_resource: AccessModelResource, dataset_resource: Resource, @@ -86,7 +113,10 @@ def _add_update_access_model_resources( return for resource_input in model_input_resources: try: - dataset_resource = Resource.objects.get(id=resource_input.resource) + # Only the access model's own dataset's resources. + dataset_resource = Resource.objects.get( + id=resource_input.resource, dataset_id=access_model.dataset_id + ) except Resource.DoesNotExist as e: raise ValueError(f"Resource with ID {resource_input.resource} does not exist.") @@ -111,7 +141,9 @@ def _update_access_model_fields( @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeAccessModelPermission] + ) def create_access_model( self, info: Info, access_model_input: AccessModelInput ) -> TypeAccessModel: @@ -131,7 +163,9 @@ def create_access_model( _add_update_access_model_resources(access_model, access_model_input.resources) return TypeAccessModel.from_django(access_model) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeAccessModelPermission] + ) def edit_access_model( self, info: Info, access_model_input: EditAccessModelInput ) -> TypeAccessModel: @@ -153,7 +187,9 @@ def edit_access_model( _add_update_access_model_resources(access_model, access_model_input.resources) return TypeAccessModel.from_django(access_model) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeAccessModelPermission] + ) def delete_access_model(self, info: Info, access_model_id: uuid.UUID) -> bool: try: access_model = AccessModel.objects.get(id=access_model_id) diff --git a/api/schema/dataset_schema.py b/api/schema/dataset_schema.py index 2aab35a2..dcc05d2a 100644 --- a/api/schema/dataset_schema.py +++ b/api/schema/dataset_schema.py @@ -44,6 +44,7 @@ from api.types.type_resource import TypeResource from api.types.type_resource_chart import TypeResourceChart from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ChangeDatasetPartPermission from api.utils.enums import ( DatasetAccessType, DatasetLicense, @@ -262,6 +263,21 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return False +class ChangePromptResourcePermission(ChangeDatasetPartPermission): + """Prompt details of a file need edit rights on the file's dataset.""" + + message = "You don't have permission to update this dataset" + + def datasets(self, kwargs: Any) -> Optional[list]: + resource_id = getattr(kwargs.get("update_input"), "resource", None) + resource = ( + Resource.objects.select_related("dataset").filter(id=resource_id).first() + if resource_id + else None + ) + return [resource.dataset] if resource else None + + @strawberry.input class DSMetadataItemType: id: str @@ -855,7 +871,7 @@ def update_prompt_metadata( return MutationResponse.success_response(TypePromptDataset.from_django(prompt_dataset)) @strawberry.mutation( - permission_classes=[IsAuthenticated], + permission_classes=[ChangePromptResourcePermission], ) @trace_resolver( name="update_prompt_resource", diff --git a/api/schema/resource_chart_schema.py b/api/schema/resource_chart_schema.py index a4e3b3b6..5eed2873 100644 --- a/api/schema/resource_chart_schema.py +++ b/api/schema/resource_chart_schema.py @@ -20,6 +20,11 @@ from api.schema.extensions import TrackModelActivity from api.types.type_resource_chart import ChartResponse, TypeResourceChart from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import AggregateType, ChartStatus, ChartTypes from api.utils.graphql_telemetry import trace_resolver from api.utils.qs_utils import ( @@ -84,12 +89,21 @@ def _visible_charts(info: Info) -> "tuple[QuerySet, QuerySet]": class Query: @strawberry_django.field def charts_details(self, info: Info, dataset_id: uuid.UUID) -> List[TypeResourceChart]: - charts = ResourceChartDetails.objects.filter(resource__dataset_id=dataset_id) + charts = ResourceChartDetails.objects.filter(resource__dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="resource__dataset__") + ) return [TypeResourceChart.from_django(chart) for chart in charts] @strawberry_django.field def resource_chart(self, info: Info, chart_details_id: uuid.UUID) -> TypeResourceChart: - chart = ResourceChartDetails.objects.get(id=chart_details_id) + chart = ResourceChartDetails.objects.select_related("resource__dataset").get( + id=chart_details_id + ) + if not can_view_dataset(info.context.user, chart.resource.dataset): + # Same answer as a missing ID. + raise ResourceChartDetails.DoesNotExist( + "ResourceChartDetails matching query does not exist." + ) return TypeResourceChart.from_django(chart) @strawberry.field( @@ -167,6 +181,36 @@ def charts_table( ) +class ChangeChartPermission(ChangeDatasetPartPermission): + """Charts belong to a resource's dataset; changing one needs edit rights on it. + + Editing a chart onto another resource needs rights on both datasets. + """ + + message = "You don't have permission to change this dataset's charts" + + def datasets(self, kwargs: Any) -> Optional[list]: + chart_input = kwargs.get("chart_input") + found = [] + for resource_id in (kwargs.get("resource"), getattr(chart_input, "resource", None)): + if resource_id: + resource = Resource.objects.select_related("dataset").filter(id=resource_id).first() + if not resource: + return None + found.append(resource.dataset) + for chart_id in (kwargs.get("chart_id"), getattr(chart_input, "chart_id", None)): + if chart_id: + chart = ( + ResourceChartDetails.objects.select_related("resource__dataset") + .filter(id=chart_id) + .first() + ) + if not chart: + return None + found.append(chart.resource.dataset) + return found + + @strawberry.input class FilterInput: column: str @@ -316,13 +360,15 @@ def _update_chart_fields( "geospatial_field", ]: if value: # Only process if value is not empty - field = ResourceSchema.objects.get(id=value) + field = ResourceSchema.objects.get(id=value, resource=resource) options[field_name] = field elif field_name == "y_axis_column": if value: # Only process if list is not empty options[field_name] = [ { - "field": ResourceSchema.objects.get(id=column.field_name), + "field": ResourceSchema.objects.get( + id=column.field_name, resource=resource + ), "label": column.label, "color": column.color, "value_mapping": _update_value_mapping(column.value_mapping), @@ -344,7 +390,7 @@ def _update_chart_fields( for filter_input in chart_input.filters: if filter_input.column: filter_dict = { - "column": ResourceSchema.objects.get(id=filter_input.column), + "column": ResourceSchema.objects.get(id=filter_input.column, resource=resource), "operator": filter_input.operator, "value": filter_input.value, } @@ -355,7 +401,9 @@ def _update_chart_fields( @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartPermission] + ) def add_resource_chart(self, info: Info, resource: uuid.UUID) -> TypeResourceChart: try: resource_obj = Resource.objects.get(id=resource) @@ -371,6 +419,7 @@ def add_resource_chart(self, info: Info, resource: uuid.UUID) -> TypeResourceCha @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeChartPermission], extensions=[ # Use TrackModelActivity to automatically track the activity # The extension will use the returned chart as the action object @@ -410,6 +459,7 @@ def create_resource_chart( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeChartPermission], extensions=[ # Track updates to charts TrackModelActivity( @@ -444,7 +494,9 @@ def edit_resource_chart(self, info: Info, chart_input: ResourceChartInput) -> Ty _update_chart_fields(chart, chart_input, resource) return TypeResourceChart.from_django(chart) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def delete_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) @@ -453,7 +505,9 @@ def delete_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: except ResourceChartDetails.DoesNotExist as e: raise ValueError(f"Resource Chart with ID {chart_id} does not exist.") - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def publish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) @@ -463,7 +517,9 @@ def publish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: except ResourceChartDetails.DoesNotExist as e: raise ValueError(f"Resource Chart with ID {chart_id} does not exist.") - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def unpublish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) diff --git a/api/schema/resource_schema.py b/api/schema/resource_schema.py index 85fbc965..41727ba6 100644 --- a/api/schema/resource_schema.py +++ b/api/schema/resource_schema.py @@ -4,7 +4,7 @@ # mypy: disable-error-code=operator # mypy: disable-error-code=valid-type -from typing import List, Optional +from typing import Any, List, Optional import strawberry import strawberry_django @@ -27,12 +27,45 @@ from api.types.type_resource import TypeResource from api.utils.constants import FORMAT_MAPPING from api.utils.data_indexing import index_resource_data +from api.utils.dataset_access import ChangeDatasetPartPermission, can_view_dataset from api.utils.file_utils import file_validation from api.utils.graphql_telemetry import trace_resolver logger = structlog.get_logger("dataspace.resource_schema") +def _dataset_from_arguments(kwargs: Any) -> Optional[Dataset]: + """The dataset a resource mutation acts on, from whichever argument it takes.""" + data = ( + kwargs.get("file_resource_input") + or kwargs.get("schema_update_input") + or kwargs.get("input") + ) + dataset_id = getattr(data, "dataset", None) + if dataset_id: + return Dataset.objects.filter(id=dataset_id).first() + resource_id = ( + kwargs.get("resource_id") + or getattr(data, "id", None) + or getattr(data, "resource", None) + or getattr(data, "resource_id", None) + ) + if not resource_id: + return None + resource = Resource.objects.select_related("dataset").filter(id=resource_id).first() + return resource.dataset if resource else None + + +class ChangeResourcePermission(ChangeDatasetPartPermission): + """Adding, changing or deleting a dataset's resources needs edit rights on the dataset.""" + + message = "You don't have permission to change this dataset's resources" + + def datasets(self, kwargs: Any) -> Optional[list]: + dataset = _dataset_from_arguments(kwargs) + return [dataset] if dataset else None + + @strawberry.input class CreateFileResourceInput: """Input type for creating a file resource.""" @@ -114,14 +147,20 @@ class Query: @trace_resolver(name="get_dataset_resources", attributes={"component": "resource"}) def dataset_resources(self, info: Info, dataset_id: uuid.UUID) -> List[TypeResource]: """Get resources for a dataset.""" - resources = Resource.objects.filter(dataset_id=dataset_id) + dataset = Dataset.objects.filter(id=dataset_id).first() + if not dataset or not can_view_dataset(info.context.user, dataset): + return [] + resources = Resource.objects.filter(dataset=dataset) return [TypeResource.from_django(resource) for resource in resources] @strawberry_django.field @trace_resolver(name="get_resource_by_id", attributes={"component": "resource"}) def resource_by_id(self, info: Info, resource_id: uuid.UUID) -> TypeResource: """Get a resource by ID.""" - resource = Resource.objects.get(id=resource_id) + resource = Resource.objects.select_related("dataset").get(id=resource_id) + if not can_view_dataset(info.context.user, resource.dataset): + # Same answer as a missing ID, so drafts' resources are not revealed. + raise Resource.DoesNotExist("Resource matching query does not exist.") return TypeResource.from_django(resource) @@ -222,6 +261,7 @@ class Mutation: @strawberry_django.mutation( handle_django_errors=False, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="created", @@ -258,6 +298,7 @@ def create_file_resources( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="created", @@ -285,6 +326,7 @@ def create_file_resource( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="updated", @@ -343,7 +385,10 @@ def update_file_resource( return TypeResource.from_django(resource) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[ChangeResourcePermission], + ) @trace_resolver(name="update_file_resource_schema", attributes={"component": "resource"}) def update_file_resource_schema( self, info: Info, schema_update_input: SchemaUpdateInput @@ -357,7 +402,10 @@ def update_file_resource_schema( _update_file_resource_schema(resource, schema_update_input.updates) return TypeResource.from_django(resource) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[ChangeResourcePermission], + ) @trace_resolver(name="reset_file_resource_schema", attributes={"component": "resource"}) def reset_file_resource_schema(self, info: Info, resource_id: uuid.UUID) -> TypeResource: """Reset file resource schema.""" @@ -372,6 +420,7 @@ def reset_file_resource_schema(self, info: Info, resource_id: uuid.UUID) -> Type @strawberry_django.mutation( handle_django_errors=False, + permission_classes=[ChangeResourcePermission], extensions=[ TrackActivity( verb="deleted", @@ -393,6 +442,7 @@ def delete_file_resource(self, info: Info, resource_id: uuid.UUID) -> bool: @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="versioned", diff --git a/api/schema/resoure_chart_image_schema.py b/api/schema/resoure_chart_image_schema.py index 6e40728a..6447a58c 100644 --- a/api/schema/resoure_chart_image_schema.py +++ b/api/schema/resoure_chart_image_schema.py @@ -1,6 +1,6 @@ import datetime import uuid -from typing import List, Optional +from typing import Any, List, Optional import strawberry import strawberry_django @@ -10,6 +10,11 @@ from api.models import Dataset, ResourceChartImage from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import ChartStatus @@ -34,6 +39,7 @@ class ResourceChartImageInputPartial: @strawberry.type(name="Query") class Query: + # Draft datasets' images are hidden by TypeResourceChartImage.get_queryset. resource_chart_images: list[TypeResourceChartImage] = strawberry_django.field() @strawberry_django.field(pagination=True) @@ -41,23 +47,51 @@ def dataset_resource_charts( self, info: Info, dataset_id: uuid.UUID ) -> List[TypeResourceChartImage]: """Get all resource chart images for a dataset.""" - images = ResourceChartImage.objects.filter(dataset_id=dataset_id) + images = ResourceChartImage.objects.filter(dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeResourceChartImage.from_django(image) for image in images] @strawberry_django.field - def resource_chart_image( - self, info: Info, image_id: uuid.UUID - ) -> TypeResourceChartImage: - try: - image = ResourceChartImage.objects.get(id=image_id) - return TypeResourceChartImage.from_django(image) - except ResourceChartImage.DoesNotExist as e: + def resource_chart_image(self, info: Info, image_id: uuid.UUID) -> TypeResourceChartImage: + image = ResourceChartImage.objects.select_related("dataset").filter(id=image_id).first() + # A hidden image gets the same answer as a missing one. + if not image or not image.dataset or not can_view_dataset(info.context.user, image.dataset): raise ValueError(f"Resource Chart Image with ID {image_id} does not exist.") + return TypeResourceChartImage.from_django(image) + + +class ChangeChartImagePermission(ChangeDatasetPartPermission): + """Chart images belong to a dataset; changing one needs edit rights on it. + + Moving an image to another dataset needs rights on both. + """ + + message = "You don't have permission to change this dataset's chart images" + + def datasets(self, kwargs: Any) -> Optional[list]: + data = kwargs.get("input") + found = [] + for dataset_id in (kwargs.get("dataset"), getattr(data, "dataset", None)): + if dataset_id: + dataset = Dataset.objects.filter(id=dataset_id).first() + if not dataset: + return None + found.append(dataset) + image_id = kwargs.get("resource_chart_image_id") or getattr(data, "id", None) + if image_id: + image = ResourceChartImage.objects.select_related("dataset").filter(id=image_id).first() + if not image or not image.dataset: + return None + found.append(image.dataset) + return found @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) def create_resource_chart_image( self, info: Info, input: ResourceChartImageInput ) -> TypeResourceChartImage: @@ -68,14 +102,15 @@ def create_resource_chart_image( raise ValueError(f"Dataset with ID {input.dataset} does not exist.") now = datetime.datetime.now() image = ResourceChartImage.objects.create( - name=input.name - or f"New resource_chart_image {now.strftime('%d %b %Y - %H:%M:%S')}", + name=input.name or f"New resource_chart_image {now.strftime('%d %b %Y - %H:%M:%S')}", dataset=dataset_obj, image=input.image, ) return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) def update_resource_chart_image( self, info: Info, input: ResourceChartImageInputPartial ) -> TypeResourceChartImage: @@ -96,10 +131,10 @@ def update_resource_chart_image( image.save() return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=True) - def add_resource_chart_image( - self, info: Info, dataset: uuid.UUID - ) -> TypeResourceChartImage: + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) + def add_resource_chart_image(self, info: Info, dataset: uuid.UUID) -> TypeResourceChartImage: """Add a new resource chart image to a dataset.""" try: dataset_obj = Dataset.objects.get(id=dataset) @@ -113,10 +148,10 @@ def add_resource_chart_image( ) return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=False) - def publish_resource_chart_image( - self, info: Info, resource_chart_image_id: uuid.UUID - ) -> bool: + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) + def publish_resource_chart_image(self, info: Info, resource_chart_image_id: uuid.UUID) -> bool: try: image = ResourceChartImage.objects.get(id=resource_chart_image_id) image.status = ChartStatus.PUBLISHED @@ -127,7 +162,9 @@ def publish_resource_chart_image( f"Resource Chart Image with ID {resource_chart_image_id} does not exist." ) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) def unpublish_resource_chart_image( self, info: Info, resource_chart_image_id: uuid.UUID ) -> bool: @@ -141,10 +178,10 @@ def unpublish_resource_chart_image( f"Resource Chart Image with ID {resource_chart_image_id} does not exist." ) - @strawberry_django.mutation(handle_django_errors=False) - def delete_resource_chart_image( - self, info: Info, resource_chart_image_id: uuid.UUID - ) -> bool: + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) + def delete_resource_chart_image(self, info: Info, resource_chart_image_id: uuid.UUID) -> bool: """Delete a resource chart image.""" try: image = ResourceChartImage.objects.get(id=resource_chart_image_id) diff --git a/api/schema/schema.py b/api/schema/schema.py index 59584dfb..1ac6b59b 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -33,6 +33,7 @@ from api.types.type_dataset import TypeTag from api.types.type_metadata import TypeMetadata from api.types.type_resource import TypeResource +from api.utils.dataset_access import viewable_datasets_q from api.utils.graphql_error_extension import ErrorFormatterExtension from api.utils.graphql_telemetry import TelemetryExtension, trace_resolver from authorization.graphql import Mutation as AuthMutation @@ -47,7 +48,9 @@ class DefaultQuery: @strawberry_django.field @trace_resolver(name="resources", attributes={"component": "default"}) def resources(self, info: Info) -> List[TypeResource]: - resources = Resource.objects.all() + resources = Resource.objects.filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeResource.from_django(resource) for resource in resources] @strawberry_django.field diff --git a/api/types/type_resource_chart_image.py b/api/types/type_resource_chart_image.py index b0d87623..92c491d5 100644 --- a/api/types/type_resource_chart_image.py +++ b/api/types/type_resource_chart_image.py @@ -1,14 +1,17 @@ # mypy: disable-error-code="valid-type" -from typing import Optional +from typing import Any, Optional import strawberry import strawberry_django +from django.db.models import QuerySet from strawberry import auto +from strawberry.types import Info from api.models import ResourceChartImage from api.types.base_type import BaseType from api.types.type_dataset import TypeDataset +from api.utils.dataset_access import viewable_datasets_q from api.utils.enums import ChartStatus @@ -32,3 +35,8 @@ class TypeResourceChartImage(BaseType): description: Optional[str] = "" dataset: Optional[TypeDataset] = None status: ChartStatusEnum = ChartStatusEnum.DRAFT + + @classmethod + def get_queryset(cls, queryset: QuerySet, info: Info, **kwargs: Any) -> QuerySet: + """Only images of datasets the caller may see.""" + return queryset.filter(viewable_datasets_q(info.context.user, prefix="dataset__")) diff --git a/api/utils/dataset_access.py b/api/utils/dataset_access.py new file mode 100644 index 00000000..2e708b90 --- /dev/null +++ b/api/utils/dataset_access.py @@ -0,0 +1,109 @@ +"""Who may see or change a dataset and the things hanging off it. + +Resources, charts, chart images and access models have no rights of their own: +anyone who may change the dataset may change them, and they are visible to +whoever may see the dataset. +""" + +from typing import Any, Optional + +from django.db.models import Q +from strawberry.permission import BasePermission +from strawberry.types import Info + +from api.models import Dataset +from api.utils.enums import DatasetStatus +from authorization.models import DatasetPermission, OrganizationMembership + + +def can_view_dataset(user: Any, dataset: Dataset) -> bool: + """Published datasets are public; drafts only for people who may see them.""" + if dataset.status == DatasetStatus.PUBLISHED.value: + return True + if not user or not user.is_authenticated: + return False + if user.is_superuser or dataset.user_id == user.id: + return True + dataset_perm = ( + DatasetPermission.objects.filter(user=user, dataset=dataset).select_related("role").first() + ) + if dataset_perm: + return bool(dataset_perm.role.can_view) + membership = ( + OrganizationMembership.objects.filter(user=user, organization_id=dataset.organization_id) + .select_related("role") + .first() + if dataset.organization_id + else None + ) + return bool(membership and membership.role.can_view) + + +def viewable_datasets_q(user: Any, prefix: str = "") -> Q: + """`can_view_dataset` as a filter, for lists. `prefix` reaches the dataset, e.g. "dataset__".""" + published = Q(**{f"{prefix}status": DatasetStatus.PUBLISHED.value}) + if not user or not user.is_authenticated: + return published + if user.is_superuser: + return Q() + shared = DatasetPermission.objects.filter(user=user, role__can_view=True).values("dataset_id") + refused = DatasetPermission.objects.filter(user=user, role__can_view=False).values("dataset_id") + orgs = OrganizationMembership.objects.filter(user=user, role__can_view=True).values( + "organization_id" + ) + return ( + published + | Q(**{f"{prefix}user_id": user.id}) + | Q(**{f"{prefix}id__in": shared}) + | (Q(**{f"{prefix}organization_id__in": orgs}) & ~Q(**{f"{prefix}id__in": refused})) + ) + + +def can_change_dataset(user: Any, dataset: Dataset) -> bool: + """Same rules as UpdateDatasetPermission in dataset_schema.""" + if not user or not user.is_authenticated: + return False + if user.is_superuser or dataset.user_id == user.id: + return True + if ( + dataset.organization_id + and OrganizationMembership.objects.filter( + user=user, + organization_id=dataset.organization_id, + role__name__in=["admin", "editor", "owner"], + ).exists() + ): + return True + dataset_perm = ( + DatasetPermission.objects.filter(user=user, dataset=dataset).select_related("role").first() + ) + return bool( + dataset_perm + and dataset_perm.role.can_change + and dataset.status == DatasetStatus.DRAFT.value + ) + + +class ChangeDatasetPartPermission(BasePermission): + """Base for mutations on something that belongs to a dataset. + + Subclasses return every dataset the call touches (the one the object is in + now and, when it is being moved, the one it goes to); the user needs edit + rights on all of them. A missing ID is refused the same way, so the answer + does not reveal whether an ID exists. + """ + + message = "You don't have permission to change this dataset" + + def datasets(self, kwargs: Any) -> Optional[list]: + """The datasets the call touches, or None if any ID does not exist.""" + raise NotImplementedError + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = getattr(info.context, "user", None) + if not user or not user.is_authenticated: + return False + datasets = self.datasets(kwargs) + if not datasets: + return False + return all(can_change_dataset(user, d) for d in datasets) From e1f6d63ac6ba2cd4251e1d551580a34da9310396 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 30 Oct 2026 03:09:40 +0530 Subject: [PATCH 27/55] refactor(api): share one dataset edit rule and limit organisation listings UpdateDatasetPermission now uses the same edit rule as resources, charts and access models. Any one grant is enough to see a draft dataset. Refused changes are logged with the operation, reason and ids only. datasets and datasetsTable list an organisation's datasets only to superusers and members whose role can view. Removes the unused ResourcePermissionGraphQL. --- api/schema/dataset_schema.py | 63 ++++++------------ api/utils/dataset_access.py | 120 +++++++++++++++++++++++------------ authorization/permissions.py | 9 --- 3 files changed, 96 insertions(+), 96 deletions(-) diff --git a/api/schema/dataset_schema.py b/api/schema/dataset_schema.py index dcc05d2a..e0b8d98a 100644 --- a/api/schema/dataset_schema.py +++ b/api/schema/dataset_schema.py @@ -44,7 +44,11 @@ from api.types.type_resource import TypeResource from api.types.type_resource_chart import TypeResourceChart from api.types.type_resource_chart_image import TypeResourceChartImage -from api.utils.dataset_access import ChangeDatasetPartPermission +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_change_dataset, + can_list_organization, +) from api.utils.enums import ( DatasetAccessType, DatasetLicense, @@ -229,38 +233,10 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: if not update_dataset_input or not hasattr(update_dataset_input, "dataset"): return False - dataset_id = update_dataset_input.dataset - - try: - dataset = Dataset.objects.get(id=dataset_id) - - # Check if user owns the dataset - if dataset.user and dataset.user == user: - return True - - # If organization-owned, check organization permissions - if dataset.organization: - # Get the roles with names 'admin' or 'editor' - admin_editor_roles = Role.objects.filter( - name__in=["admin", "editor", "owner"] - ).values_list("id", flat=True) - - # Check if user is a member of the dataset's organization with appropriate role - org_member = OrganizationMembership.objects.filter( - user=user, - organization=dataset.organization, - role__id__in=admin_editor_roles, - ).exists() - - if org_member: - return True - - # Check dataset-specific permissions - dataset_perm = DatasetPermission.objects.filter(user=user, dataset=dataset).first() - return dataset_perm and dataset_perm.role.can_change and dataset.status == DatasetStatus.DRAFT.value # type: ignore - - except Dataset.DoesNotExist: + dataset = Dataset.objects.filter(id=update_dataset_input.dataset).first() + if not dataset: return False + return can_change_dataset(user, dataset) class ChangePromptResourcePermission(ChangeDatasetPartPermission): @@ -451,7 +427,11 @@ def _visible_datasets(info: Info, include_public: Optional[bool] = False) -> Que organization = info.context.context.get("organization") user = info.context.user if organization: - queryset = Dataset.objects.filter(organization=organization) + queryset = ( + Dataset.objects.filter(organization=organization) + if can_list_organization(user, organization) + else Dataset.objects.none() + ) elif not user.is_authenticated: queryset = Dataset.objects.none() elif user.is_superuser: @@ -480,24 +460,17 @@ def datasets( user = info.context.user if organization: - queryset = Dataset.objects.filter(organization=organization) + # Only members whose role can view may list an organisation's datasets + if can_list_organization(user, organization): + queryset = Dataset.objects.filter(organization=organization) + else: + queryset = Dataset.objects.none() else: # If user is authenticated if user.is_authenticated: # If user is superuser, show all datasets if user.is_superuser: queryset = Dataset.objects.all() - elif organization: - # Check id user has access to organization - org_member = OrganizationMembership.objects.get( - user=user, organization=organization - ) - if org_member.exists() and org_member.role.can_view: # type: ignore - # Show only datasets from current organization - queryset = Dataset.objects.filter(organization=organization) - else: - # if user is not a member of the organization, return empty queryset - queryset = Dataset.objects.none() else: # For non-organization authenticated users, only owned datasets queryset = Dataset.objects.filter(user=user, organization=None) diff --git a/api/utils/dataset_access.py b/api/utils/dataset_access.py index 2e708b90..e8eb8943 100644 --- a/api/utils/dataset_access.py +++ b/api/utils/dataset_access.py @@ -2,11 +2,13 @@ Resources, charts, chart images and access models have no rights of their own: anyone who may change the dataset may change them, and they are visible to -whoever may see the dataset. +whoever may see the dataset. The organisation is always the dataset's own; +nothing here reads the request's organisation header. """ from typing import Any, Optional +import structlog from django.db.models import Q from strawberry.permission import BasePermission from strawberry.types import Info @@ -15,28 +17,55 @@ from api.utils.enums import DatasetStatus from authorization.models import DatasetPermission, OrganizationMembership +logger = structlog.get_logger(__name__) -def can_view_dataset(user: Any, dataset: Dataset) -> bool: - """Published datasets are public; drafts only for people who may see them.""" - if dataset.status == DatasetStatus.PUBLISHED.value: - return True +# Organisation roles that may edit the organisation's datasets, by name. +DATASET_EDITOR_ROLE_NAMES = ["admin", "editor", "owner"] + + +def can_change_dataset(user: Any, dataset: Dataset) -> bool: + """The one dataset edit rule, also used by UpdateDatasetPermission.""" if not user or not user.is_authenticated: return False if user.is_superuser or dataset.user_id == user.id: return True + if ( + dataset.organization_id + and OrganizationMembership.objects.filter( + user=user, + organization_id=dataset.organization_id, + role__name__in=DATASET_EDITOR_ROLE_NAMES, + ).exists() + ): + return True dataset_perm = ( DatasetPermission.objects.filter(user=user, dataset=dataset).select_related("role").first() ) - if dataset_perm: - return bool(dataset_perm.role.can_view) - membership = ( - OrganizationMembership.objects.filter(user=user, organization_id=dataset.organization_id) - .select_related("role") - .first() - if dataset.organization_id - else None + return bool( + dataset_perm + and dataset_perm.role.can_change + and dataset.status == DatasetStatus.DRAFT.value ) - return bool(membership and membership.role.can_view) + + +def can_view_dataset(user: Any, dataset: Dataset) -> bool: + """Published datasets are public; otherwise any one grant is enough.""" + if dataset.status == DatasetStatus.PUBLISHED.value: + return True + if not user or not user.is_authenticated: + return False + if user.is_superuser or dataset.user_id == user.id: + return True + if ( + dataset.organization_id + and OrganizationMembership.objects.filter( + user=user, organization_id=dataset.organization_id, role__can_view=True + ).exists() + ): + return True + if DatasetPermission.objects.filter(user=user, dataset=dataset, role__can_view=True).exists(): + return True + return can_change_dataset(user, dataset) def viewable_datasets_q(user: Any, prefix: str = "") -> Q: @@ -46,42 +75,46 @@ def viewable_datasets_q(user: Any, prefix: str = "") -> Q: return published if user.is_superuser: return Q() - shared = DatasetPermission.objects.filter(user=user, role__can_view=True).values("dataset_id") - refused = DatasetPermission.objects.filter(user=user, role__can_view=False).values("dataset_id") - orgs = OrganizationMembership.objects.filter(user=user, role__can_view=True).values( - "organization_id" + orgs = OrganizationMembership.objects.filter(user=user).filter( + Q(role__can_view=True) | Q(role__name__in=DATASET_EDITOR_ROLE_NAMES) + ) + shared = DatasetPermission.objects.filter(user=user, role__can_view=True) + draft_editors = DatasetPermission.objects.filter( + user=user, role__can_change=True, dataset__status=DatasetStatus.DRAFT.value ) return ( published | Q(**{f"{prefix}user_id": user.id}) - | Q(**{f"{prefix}id__in": shared}) - | (Q(**{f"{prefix}organization_id__in": orgs}) & ~Q(**{f"{prefix}id__in": refused})) + | Q(**{f"{prefix}organization_id__in": orgs.values("organization_id")}) + | Q(**{f"{prefix}id__in": shared.values("dataset_id")}) + | Q(**{f"{prefix}id__in": draft_editors.values("dataset_id")}) ) -def can_change_dataset(user: Any, dataset: Dataset) -> bool: - """Same rules as UpdateDatasetPermission in dataset_schema.""" +def can_list_organization(user: Any, organization: Any) -> bool: + """Whether this user may list an organisation's datasets, drafts included.""" if not user or not user.is_authenticated: return False - if user.is_superuser or dataset.user_id == user.id: - return True - if ( - dataset.organization_id - and OrganizationMembership.objects.filter( - user=user, - organization_id=dataset.organization_id, - role__name__in=["admin", "editor", "owner"], - ).exists() - ): + if user.is_superuser: return True - dataset_perm = ( - DatasetPermission.objects.filter(user=user, dataset=dataset).select_related("role").first() - ) - return bool( - dataset_perm - and dataset_perm.role.can_change - and dataset.status == DatasetStatus.DRAFT.value + return OrganizationMembership.objects.filter( + user=user, organization=organization, role__can_view=True + ).exists() + + +def log_access_denied( + info: Info, user: Any, reason: str, dataset: Optional[Dataset] = None +) -> bool: + """Record one warning for a refused request, and return False.""" + logger.warning( + "dataset access denied", + operation=info.field_name, + user_id=getattr(user, "id", None), + reason=reason, + dataset_id=str(dataset.id) if dataset else None, + organization_id=dataset.organization_id if dataset else None, ) + return False class ChangeDatasetPartPermission(BasePermission): @@ -102,8 +135,11 @@ def datasets(self, kwargs: Any) -> Optional[list]: def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: user = getattr(info.context, "user", None) if not user or not user.is_authenticated: - return False + return log_access_denied(info, user, "not_signed_in") datasets = self.datasets(kwargs) if not datasets: - return False - return all(can_change_dataset(user, d) for d in datasets) + return log_access_denied(info, user, "target_not_found") + for dataset in datasets: + if not can_change_dataset(user, dataset): + return log_access_denied(info, user, "not_allowed", dataset) + return True diff --git a/authorization/permissions.py b/authorization/permissions.py index fb27f87f..03cc90ca 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -392,15 +392,6 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True -class ResourcePermissionGraphQL(HasOrganizationRoleGraphQL): # type: ignore[misc] - """ - Permission class specifically for Resource objects. - """ - - def _get_organization(self, obj: Any) -> Optional[Organization]: - return obj.dataset.organization if hasattr(obj, "dataset") else None # type: ignore[attr-defined,no-any-return] - - class PublishDatasetPermission(BasePermission): """Permission class for publishing a dataset. Checks if the user has permission to publish the dataset. From ba39243d556a6b406abbbdd87d6853ebbb7940fd Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Fri, 30 Oct 2026 12:00:56 +0530 Subject: [PATCH 28/55] test(api): cover dataset, resource, chart and access model permissions --- tests/schema/test_dataset_access_rules.py | 282 +++++++++++++ tests/schema/test_dataset_part_permissions.py | 372 ++++++++++++++++++ tests/schema/test_resource_permissions.py | 183 +++++++++ 3 files changed, 837 insertions(+) create mode 100644 tests/schema/test_dataset_access_rules.py create mode 100644 tests/schema/test_dataset_part_permissions.py create mode 100644 tests/schema/test_resource_permissions.py diff --git a/tests/schema/test_dataset_access_rules.py b/tests/schema/test_dataset_access_rules.py new file mode 100644 index 00000000..2ad1e302 --- /dev/null +++ b/tests/schema/test_dataset_access_rules.py @@ -0,0 +1,282 @@ +"""One dataset edit rule, any-grant view rule, organisation listings for +members only, and refused changes are logged.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Dataset, Organization, Resource +from api.schema.dataset_schema import UpdateDatasetPermission +from api.schema.schema import schema +from api.utils.dataset_access import ( + can_change_dataset, + can_view_dataset, + viewable_datasets_q, +) +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + + +def ctx(user, org=None): + return types.SimpleNamespace( + user=user, + context={"organization": org, "auth_token": "CANARY-TOKEN"} if org else {}, + request=None, + response=None, + ) + + +def info(user): + return types.SimpleNamespace(context=ctx(user), field_name="updateDataset") + + +def run(user, query, org=None, **variables): + return schema.execute_sync(query, variable_values=variables, context_value=ctx(user, org)) + + +@pytest.fixture +def w(db): + roles = { + "admin": Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ), + "editor": Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True), + "viewer": Role.objects.create(name="viewer", can_view=True), + "manager": Role.objects.create( + name="manager", can_view=True, can_change=True, can_delete=True + ), + "ds_editor": Role.objects.create(name="dataset-editor", can_view=True, can_change=True), + "blind": Role.objects.create(name="blind", can_view=False), + } + org = Organization.objects.create(name="Org", description="o", slug="org") + other = Organization.objects.create(name="Other", description="o", slug="other") + names = [ + "owner", + "admin", + "editor", + "viewer", + "manager", + "other_admin", + "stranger", + "dp_change", + "dp_elsewhere", + "viewer_blind_dp", + "solo", + ] + u = { + n: User.objects.create(username=f"u-{n}-name", email=f"{n}@example.org", keycloak_id=n) + for n in names + } + u["superuser"] = User.objects.create(username="su", keycloak_id="su", is_superuser=True) + for n, r, o in [ + ("admin", "admin", org), + ("editor", "editor", org), + ("viewer", "viewer", org), + ("manager", "manager", org), + ("viewer_blind_dp", "viewer", org), + ("other_admin", "admin", other), + ]: + OrganizationMembership.objects.create(user=u[n], organization=o, role=roles[r]) + + def ds(slug, status="DRAFT", **kw): + return Dataset.objects.create(title=slug, slug=slug, description="d", status=status, **kw) + + d = types.SimpleNamespace( + draft=ds("draft", user=u["owner"], organization=org), + published=ds("pub", "PUBLISHED", user=u["owner"], organization=org), + archived=ds("arch", "ARCHIVED", user=u["owner"], organization=org), + solo=ds("solo", user=u["solo"]), + ) + for x in (d.draft, d.published, d.archived): + DatasetPermission.objects.create(user=u["dp_change"], dataset=x, role=roles["ds_editor"]) + DatasetPermission.objects.create( + user=u["dp_elsewhere"], dataset=d.solo, role=roles["ds_editor"] + ) + DatasetPermission.objects.create( + user=u["viewer_blind_dp"], dataset=d.draft, role=roles["blind"] + ) + return types.SimpleNamespace(u=u, d=d, org=org, other=other) + + +# ---- the edit rule --------------------------------------------------------- + +EDIT = { + # user: (draft, published, archived) + "owner": (True, True, True), + "admin": (True, True, True), + "editor": (True, True, True), + "viewer": (False, False, False), + "manager": (False, False, False), # org edit rights follow the role name + "other_admin": (False, False, False), + "stranger": (False, False, False), + "dp_change": (True, False, False), + "dp_elsewhere": (False, False, False), + "superuser": (True, True, True), +} + + +@pytest.mark.parametrize("who", EDIT) +def test_edit_rule(w, who): + got = tuple(can_change_dataset(w.u[who], x) for x in (w.d.draft, w.d.published, w.d.archived)) + assert got == EDIT[who] + + +@pytest.mark.parametrize("who", EDIT) +def test_update_dataset_permission_uses_edit_rule(w, who): + perm = UpdateDatasetPermission() + for x, expected in zip((w.d.draft, w.d.published, w.d.archived), EDIT[who]): + for key in ("update_dataset_input", "update_metadata_input", "update_input"): + got = perm.has_permission( + None, info(w.u[who]), **{key: types.SimpleNamespace(dataset=x.id)} + ) + assert bool(got) == expected, (who, x.status, key) + + +def test_update_dataset_permission_edges(w): + perm = UpdateDatasetPermission() + editor, su = w.u["editor"], w.u["superuser"] + unknown = types.SimpleNamespace(dataset=uuid.uuid4()) + assert not perm.has_permission(None, info(AnonymousUser()), update_input=unknown) + assert not perm.has_permission(None, info(editor)) + assert not perm.has_permission(None, info(editor), update_input=types.SimpleNamespace()) + assert not perm.has_permission(None, info(editor), update_input=unknown) + # Superusers still pass before the lookup, as before. + assert perm.has_permission(None, info(su)) + assert perm.has_permission(None, info(su), update_input=unknown) + + +def test_null_owner_and_org_dataset(w): + orphan = Dataset.objects.create(title="o", slug="orphan", description="d") + DatasetPermission.objects.create( + user=w.u["dp_change"], dataset=orphan, role=Role.objects.get(name="dataset-editor") + ) + assert not can_change_dataset(w.u["editor"], orphan) + assert not can_view_dataset(w.u["stranger"], orphan) + assert can_change_dataset(w.u["dp_change"], orphan) + assert can_change_dataset(w.u["superuser"], orphan) + + +# ---- the view rule --------------------------------------------------------- + +VIEW_DRAFT = { + "owner": True, + "admin": True, + "editor": True, + "viewer": True, + "manager": True, + "other_admin": False, + "stranger": False, + "dp_change": True, + "dp_elsewhere": False, + "viewer_blind_dp": True, # any one grant is enough + "superuser": True, +} + + +@pytest.mark.parametrize("who", VIEW_DRAFT) +def test_view_rule_matches_list_filter(w, who): + user = w.u[who] + for x in (w.d.draft, w.d.archived): + expected = VIEW_DRAFT[who] + assert can_view_dataset(user, x) == expected, (who, x.status) + listed = Dataset.objects.filter(viewable_datasets_q(user), id=x.id).exists() + assert listed == expected, (who, x.status) + assert can_view_dataset(user, w.d.published) + + +def test_anonymous_view_no_queries(w, django_assert_num_queries): + with django_assert_num_queries(0): + assert can_view_dataset(AnonymousUser(), w.d.published) + assert not can_view_dataset(AnonymousUser(), w.d.draft) + assert not can_change_dataset(AnonymousUser(), w.d.draft) + + +# ---- the organisation header ---------------------------------------------- + +DATASETS = "query ($p: Boolean) { datasets(includePublic: $p) { id } }" +TABLE = "query { datasetsTable(limit: 50) { data { id } totalItemsCount } }" +NESTED = "query { datasets { resources { id name } } }" + + +def ids(r, field="datasets"): + assert r.errors is None, r.errors + rows = r.data[field]["data"] if field == "datasetsTable" else r.data[field] + return {row["id"] for row in rows} + + +@pytest.mark.parametrize("who", ["anon", "stranger", "other_admin"]) +def test_header_does_not_list_another_orgs_drafts(w, who): + user = AnonymousUser() if who == "anon" else w.u[who] + Resource.objects.create(name="secret.csv", dataset=w.d.draft) + assert ids(run(user, DATASETS, org=w.org)) == set() + assert ids(run(user, TABLE, org=w.org), "datasetsTable") == set() + r = run(user, NESTED, org=w.org) + assert "secret.csv" not in str(r.data) and r.errors is None + # includePublic still adds published datasets, and only those. + assert ids(run(user, DATASETS, org=w.org, p=True)) == {str(w.d.published.id)} + + +@pytest.mark.parametrize("who", ["viewer", "editor", "admin", "superuser"]) +def test_members_list_their_orgs_datasets(w, who): + expected = {str(x.id) for x in (w.d.draft, w.d.published, w.d.archived)} + assert ids(run(w.u[who], DATASETS, org=w.org)) == expected + assert ids(run(w.u[who], TABLE, org=w.org), "datasetsTable") == expected + + +def test_no_header_lists_own_solo_datasets(w): + assert ids(run(w.u["solo"], DATASETS)) == {str(w.d.solo.id)} + + +# ---- refusals are logged --------------------------------------------------- + +RENAME = """mutation ($id: UUID!) { + updateFileResource(fileResourceInput: {id: $id, name: "x"}) { ... on TypeResource { name } } +}""" + + +def denial_records(caplog): + out = [] + for rec in caplog.records: + msg = rec.msg if isinstance(rec.msg, dict) else {} + if msg.get("event") == "dataset access denied": + out.append(msg) + return out + + +@pytest.mark.parametrize( + "who,reason", + [("anon", "not_signed_in"), ("other_admin", "not_allowed"), ("unknown", "target_not_found")], +) +def test_refusal_logged_once_without_personal_data(w, caplog, who, reason): + res = Resource.objects.create(name="f.csv", dataset=w.d.draft) + user = AnonymousUser() if who == "anon" else w.u["other_admin"] + rid = uuid.uuid4() if who == "unknown" else res.id + caplog.set_level("WARNING") + r = run(user, RENAME, org=w.org, id=str(rid)) + assert r.errors + res.refresh_from_db() + assert res.name == "f.csv" + recs = denial_records(caplog) + assert len(recs) == 1 + assert recs[0]["reason"] == reason + assert recs[0]["operation"] == "updateFileResource" + assert recs[0]["user_id"] == getattr(user, "id", None) + for rec in caplog.records: + text = rec.getMessage() + str(rec.msg) + for secret in ("CANARY-TOKEN", "other_admin@example.org", "u-other_admin-name"): + assert secret not in text + + +def test_allowed_change_logs_nothing(w, caplog): + res = Resource.objects.create(name="f.csv", dataset=w.d.draft) + caplog.set_level("WARNING") + r = run(w.u["editor"], RENAME, org=w.org, id=str(res.id)) + assert r.errors is None, r.errors + assert denial_records(caplog) == [] + + +def test_old_resource_permission_class_removed(): + import authorization.permissions as p + + assert not hasattr(p, "ResourcePermissionGraphQL") diff --git a/tests/schema/test_dataset_part_permissions.py b/tests/schema/test_dataset_part_permissions.py new file mode 100644 index 00000000..842ede7f --- /dev/null +++ b/tests/schema/test_dataset_part_permissions.py @@ -0,0 +1,372 @@ +"""Charts, chart images, access models and prompt file +details need edit rights on their dataset; drafts' ones are hidden from people +who cannot see the draft.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.test import RequestFactory + +from api.models import ( + AccessModel, + Dataset, + Organization, + PromptResource, + Resource, + ResourceChartDetails, + ResourceChartImage, + ResourceSchema, +) +from api.schema.schema import schema +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + + +def run(user, query, **variables): + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def world(db): + org = Organization.objects.create(name="Org", description="o", slug="org") + other_org = Organization.objects.create(name="Other", description="o", slug="other-org") + editor_role = Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True) + viewer_role = Role.objects.create(name="viewer", can_view=True) + users = { + name: User.objects.create(username=name, keycloak_id=name) + for name in ["owner", "editor", "viewer", "outsider", "shared"] + } + users["admin"] = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + OrganizationMembership.objects.create(user=users["editor"], organization=org, role=editor_role) + OrganizationMembership.objects.create(user=users["viewer"], organization=org, role=viewer_role) + OrganizationMembership.objects.create( + user=users["outsider"], organization=other_org, role=editor_role + ) + draft = Dataset.objects.create( + title="Draft", slug="draft", description="d", user=users["owner"], organization=org + ) + published = Dataset.objects.create( + title="Pub", + slug="pub", + description="d", + user=users["owner"], + organization=org, + status="PUBLISHED", + ) + # The outsider's own dataset, for "move it into my dataset" attempts. + theirs = Dataset.objects.create( + title="Theirs", + slug="theirs", + description="d", + user=users["outsider"], + organization=other_org, + ) + DatasetPermission.objects.create(user=users["shared"], dataset=draft, role=editor_role) + draft_res = Resource.objects.create(name="d.csv", dataset=draft) + pub_res = Resource.objects.create(name="p.csv", dataset=published) + their_res = Resource.objects.create(name="t.csv", dataset=theirs) + return types.SimpleNamespace( + users=users, + draft=draft, + published=published, + theirs=theirs, + draft_res=draft_res, + pub_res=pub_res, + their_res=their_res, + draft_chart=ResourceChartDetails.objects.create(name="dc", resource=draft_res), + pub_chart=ResourceChartDetails.objects.create(name="pc", resource=pub_res), + draft_image=ResourceChartImage.objects.create(name="di", dataset=draft), + pub_image=ResourceChartImage.objects.create(name="pi", dataset=published), + draft_am=AccessModel.objects.create(name="da", dataset=draft), + pub_am=AccessModel.objects.create(name="pa", dataset=published), + ) + + +def user_for(world, who): + return AnonymousUser() if who == "anonymous" else world.users[who] + + +def denied(result): + """A permission refusal, as a top-level error or a handled OperationInfo.""" + text = str(result.errors) if result.errors else str(result.data) + return "permission" in text + + +def u(ids): + return {k: str(v) for k, v in ids.items()} + + +CHART = "... on TypeResourceChart { id } ... on OperationInfo { messages { message } }" +IMAGE = "... on TypeResourceChartImage { id } ... on OperationInfo { messages { message } }" +AM = "... on TypeAccessModel { id } ... on OperationInfo { messages { message } }" + +# (query, variables from world, check the object is untouched afterwards) +MUTATIONS = { + "addResourceChart": ( + f"mutation ($r: UUID!) {{ addResourceChart(resource: $r) {{ {CHART} }} }}", + lambda w: {"r": w.pub_res.id}, + ), + "createResourceChart": ( + f"mutation ($r: UUID!) {{ createResourceChart(chartInput: {{resource: $r}}) {{ {CHART} }} }}", + lambda w: {"r": w.pub_res.id}, + ), + "editResourceChart": ( + f"""mutation ($c: UUID!, $r: UUID!) {{ + editResourceChart(chartInput: {{chartId: $c, resource: $r, name: "x"}}) {{ {CHART} }} + }}""", + lambda w: {"c": w.pub_chart.id, "r": w.pub_res.id}, + ), + "deleteResourceChart": ( + "mutation ($c: UUID!) { deleteResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "publishResourceChart": ( + "mutation ($c: UUID!) { publishResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "unpublishResourceChart": ( + "mutation ($c: UUID!) { unpublishResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "createResourceChartImage": ( + f"mutation ($d: UUID!) {{ createResourceChartImage(input: {{dataset: $d}}) {{ {IMAGE} }} }}", + lambda w: {"d": w.published.id}, + ), + "addResourceChartImage": ( + f"mutation ($d: UUID!) {{ addResourceChartImage(dataset: $d) {{ {IMAGE} }} }}", + lambda w: {"d": w.published.id}, + ), + "updateResourceChartImage": ( + f"""mutation ($i: UUID!, $d: UUID!) {{ + updateResourceChartImage(input: {{id: $i, dataset: $d, name: "x"}}) {{ {IMAGE} }} + }}""", + lambda w: {"i": w.pub_image.id, "d": w.published.id}, + ), + "publishResourceChartImage": ( + "mutation ($i: UUID!) { publishResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "unpublishResourceChartImage": ( + "mutation ($i: UUID!) { unpublishResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "deleteResourceChartImage": ( + "mutation ($i: UUID!) { deleteResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "createAccessModel": ( + f"""mutation ($d: UUID!) {{ + createAccessModel(accessModelInput: {{dataset: $d, name: "x", description: null, type: PUBLIC, resources: []}}) + {{ {AM} }} + }}""", + lambda w: {"d": w.published.id}, + ), + "editAccessModel": ( + f"""mutation ($a: UUID!, $d: UUID!) {{ + editAccessModel(accessModelInput: {{ + accessModelId: $a, dataset: $d, name: "x", description: null, type: null, resources: null + }}) {{ {AM} }} + }}""", + lambda w: {"a": w.pub_am.id, "d": w.published.id}, + ), + "deleteAccessModel": ( + "mutation ($a: UUID!) { deleteAccessModel(accessModelId: $a) }", + lambda w: {"a": w.pub_am.id}, + ), + "updatePromptResource": ( + """mutation ($r: UUID!) { + updatePromptResource(updateInput: {resource: $r, promptCount: 5}) { success } + }""", + lambda w: {"r": w.draft_res.id}, + ), +} + + +def untouched(w): + w.pub_chart.refresh_from_db() + w.pub_image.refresh_from_db() + w.pub_am.refresh_from_db() + return ( + w.pub_chart.name == "pc" + and w.pub_chart.status == "DRAFT" + and w.pub_image.name == "pi" + and w.pub_am.name == "pa" + and ResourceChartDetails.objects.count() == 2 + and ResourceChartImage.objects.count() == 2 + and AccessModel.objects.count() == 2 + and not PromptResource.objects.exists() + ) + + +@pytest.mark.parametrize("who", ["anonymous", "outsider", "viewer"]) +@pytest.mark.parametrize("name", list(MUTATIONS)) +def test_people_without_edit_rights_are_refused(world, who, name): + query, variables = MUTATIONS[name] + r = run(user_for(world, who), query, **u(variables(world))) + assert denied(r), (who, name, r.errors, r.data) + assert untouched(world) + + +@pytest.mark.parametrize("who", ["owner", "editor", "admin"]) +@pytest.mark.parametrize("name", list(MUTATIONS)) +def test_people_with_edit_rights_are_let_through(world, who, name): + query, variables = MUTATIONS[name] + r = run(world.users[who], query, **u(variables(world))) + assert not denied(r), (who, name, r.errors, r.data) + + +@pytest.mark.parametrize( + "query", + [ + "mutation ($i: UUID!) { deleteResourceChart(chartId: $i) }", + "mutation ($i: UUID!) { deleteResourceChartImage(resourceChartImageId: $i) }", + "mutation ($i: UUID!) { deleteAccessModel(accessModelId: $i) }", + ], +) +def test_missing_id_is_refused_like_a_forbidden_one(world, query): + r = run(AnonymousUser(), query, i=str(uuid.uuid4())) + assert denied(r) + assert "does not exist" not in str(r.errors) + + +# Moving something out of a dataset you cannot edit, into one you can. + + +def test_cannot_move_someone_elses_chart_onto_my_resource(world): + query, _ = MUTATIONS["editResourceChart"] + r = run(world.users["outsider"], query, c=str(world.pub_chart.id), r=str(world.their_res.id)) + assert denied(r) + world.pub_chart.refresh_from_db() + assert world.pub_chart.resource_id == world.pub_res.id + + +def test_cannot_move_someone_elses_chart_image_into_my_dataset(world): + query, _ = MUTATIONS["updateResourceChartImage"] + r = run(world.users["outsider"], query, i=str(world.pub_image.id), d=str(world.theirs.id)) + assert denied(r) + world.pub_image.refresh_from_db() + assert world.pub_image.dataset_id == world.published.id + + +def test_cannot_take_someone_elses_access_model_by_naming_my_dataset(world): + query, _ = MUTATIONS["editAccessModel"] + r = run(world.users["outsider"], query, a=str(world.pub_am.id), d=str(world.theirs.id)) + assert denied(r) + + +def test_access_model_cannot_include_another_datasets_resource(world): + r = run( + world.users["outsider"], + f"""mutation ($d: UUID!, $r: UUID!) {{ + createAccessModel(accessModelInput: {{ + dataset: $d, name: "x", description: null, type: PUBLIC, + resources: [{{resource: $r, fields: []}}] + }}) {{ {AM} }} + }}""", + d=str(world.theirs.id), + r=str(world.draft_res.id), + ) + assert "does not exist" in str(r.errors or r.data) + assert not AccessModel.objects.filter(accessmodelresource__resource=world.draft_res).exists() + + +def test_chart_cannot_use_another_resources_column(world): + column = ResourceSchema.objects.create( + resource=world.draft_res, field_name="secret", format="STRING" + ) + r = run( + world.users["outsider"], + f"""mutation ($r: UUID!, $col: String!) {{ + createResourceChart(chartInput: {{resource: $r, options: {{xAxisColumn: $col}}}}) + {{ {CHART} }} + }}""", + r=str(world.their_res.id), + col=str(column.id), + ) + assert "TypeResourceChart" not in str(r.data) or "does not exist" in str(r.data) + assert not any( + "secret" in str(c.options) + for c in ResourceChartDetails.objects.filter(resource=world.their_res) + ) + + +# Queries. + +LISTS = { + "chartsDetails": ("query ($d: UUID!) { chartsDetails(datasetId: $d) { id } }", "draft_chart"), + "datasetResourceCharts": ( + "query ($d: UUID!) { datasetResourceCharts(datasetId: $d) { id } }", + "draft_image", + ), + "accessModelResources": ( + "query ($d: UUID!) { accessModelResources(datasetId: $d) { id } }", + "draft_am", + ), +} +ONES = { + "resourceChart": ("query ($i: UUID!) { resourceChart(chartDetailsId: $i) { id } }", "chart"), + "resourceChartImage": ("query ($i: UUID!) { resourceChartImage(imageId: $i) { id } }", "image"), + "accessModel": ("query ($i: UUID!) { accessModel(accessModelId: $i) { id } }", "am"), +} + + +@pytest.mark.parametrize("name", list(LISTS)) +def test_published_lists_stay_public(world, name): + query, _ = LISTS[name] + r = run(AnonymousUser(), query, d=str(world.published.id)) + assert r.errors is None and len(r.data[name]) == 1 + + +@pytest.mark.parametrize("name", list(ONES)) +def test_published_items_stay_public(world, name): + query, attr = ONES[name] + r = run(AnonymousUser(), query, i=str(getattr(world, f"pub_{attr}").id)) + assert r.errors is None and r.data[name]["id"] + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +@pytest.mark.parametrize("name", list(LISTS)) +def test_draft_lists_are_hidden(world, who, name): + query, _ = LISTS[name] + r = run(user_for(world, who), query, d=str(world.draft.id)) + assert r.errors is None and r.data[name] == [] + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +@pytest.mark.parametrize("name", list(ONES)) +def test_draft_items_are_hidden(world, who, name): + query, attr = ONES[name] + r = run(user_for(world, who), query, i=str(getattr(world, f"draft_{attr}").id)) + assert r.errors and "does not exist" in str(r.errors[0].message) + + +@pytest.mark.parametrize("who", ["owner", "editor", "viewer", "shared", "admin"]) +def test_drafts_visible_to_people_who_can_see_the_draft(world, who): + user = world.users[who] + for name, (query, _) in LISTS.items(): + r = run(user, query, d=str(world.draft.id)) + assert r.errors is None and len(r.data[name]) == 1, (who, name, r.errors) + for name, (query, attr) in ONES.items(): + r = run(user, query, i=str(getattr(world, f"draft_{attr}").id)) + assert r.errors is None, (who, name, r.errors) + + +def test_all_chart_images_list_hides_drafts(world): + query = "{ resourceChartImages { name } }" + r = run(AnonymousUser(), query) + assert r.errors is None and r.data["resourceChartImages"] == [{"name": "pi"}] + r = run(world.users["owner"], query) + assert sorted(i["name"] for i in r.data["resourceChartImages"]) == ["di", "pi"] + + +def test_all_chart_images_list_still_filters(world): + r = run( + world.users["owner"], + 'query { resourceChartImages(filters: {name: "di"}) { name } }', + ) + assert r.errors is None and r.data["resourceChartImages"] == [{"name": "di"}] diff --git a/tests/schema/test_resource_permissions.py b/tests/schema/test_resource_permissions.py new file mode 100644 index 00000000..72e86a66 --- /dev/null +++ b/tests/schema/test_resource_permissions.py @@ -0,0 +1,183 @@ +"""Resource mutations need edit rights on the dataset; drafts' resources are +visible only to people who can see the draft.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.test import RequestFactory + +from api.models import Dataset, Organization, Resource +from api.schema.schema import schema +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + +DENIED = "You don't have permission to change this dataset's resources" + + +def run(user, query, **variables): + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def world(db): + org = Organization.objects.create(name="Org", description="o", slug="org") + other_org = Organization.objects.create(name="Other", description="o", slug="other-org") + editor_role = Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True) + viewer_role = Role.objects.create(name="viewer", can_view=True) + users = { + name: User.objects.create(username=name, keycloak_id=name) + for name in ["owner", "editor", "viewer", "outsider", "shared"] + } + users["admin"] = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + OrganizationMembership.objects.create(user=users["editor"], organization=org, role=editor_role) + OrganizationMembership.objects.create(user=users["viewer"], organization=org, role=viewer_role) + OrganizationMembership.objects.create( + user=users["outsider"], organization=other_org, role=editor_role + ) + draft = Dataset.objects.create( + title="Draft", slug="draft", description="d", user=users["owner"], organization=org + ) + published = Dataset.objects.create( + title="Pub", + slug="pub", + description="d", + user=users["owner"], + organization=org, + status="PUBLISHED", + ) + DatasetPermission.objects.create(user=users["shared"], dataset=draft, role=editor_role) + return types.SimpleNamespace( + users=users, + draft=draft, + published=published, + draft_res=Resource.objects.create(name="d.csv", dataset=draft), + pub_res=Resource.objects.create(name="p.csv", dataset=published), + ) + + +DELETE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" +RENAME = """mutation ($id: UUID!) { + updateFileResource(fileResourceInput: {id: $id, name: "renamed"}) { ... on TypeResource { name } } +}""" +CREATE_EMPTY = """mutation ($ds: UUID!) { + createFileResource(fileResourceInput: {dataset: $ds}) { ... on TypeResource { id } } +}""" +CREATE_FILES = """mutation ($ds: UUID!) { + createFileResources(fileResourceInput: {dataset: $ds, files: []}) { id } +}""" +RESET_SCHEMA = """mutation ($id: UUID!) { + resetFileResourceSchema(resourceId: $id) { ... on TypeResource { id } } +}""" +UPDATE_SCHEMA = """mutation ($id: UUID!) { + updateFileResourceSchema(schemaUpdateInput: {resource: $id, updates: []}) { + ... on TypeResource { id } + } +}""" +MAJOR = """mutation ($id: UUID!) { + createMajorVersion(input: {resourceId: $id, description: "x"}) { ... on TypeResource { id } } +}""" + + +def denied(result): + """Either a top-level GraphQL error or a handled OperationInfo, both with DENIED.""" + if result.errors: + return any(DENIED in str(e.message) for e in result.errors) + payload = next(iter(result.data.values())) + return bool(payload) and DENIED in str(payload.get("messages", payload)) + + +@pytest.mark.parametrize("who", ["anonymous", "outsider", "viewer"]) +@pytest.mark.parametrize( + "query,arg", + [ + (DELETE, "res"), + (RENAME, "res"), + (RESET_SCHEMA, "res"), + (UPDATE_SCHEMA, "res"), + (MAJOR, "res"), + (CREATE_EMPTY, "ds"), + (CREATE_FILES, "ds"), + ], +) +def test_people_without_edit_rights_are_refused(world, who, query, arg): + user = AnonymousUser() if who == "anonymous" else world.users[who] + variables = {"id": str(world.pub_res.id)} if arg == "res" else {"ds": str(world.published.id)} + if query in (RENAME, UPDATE_SCHEMA, RESET_SCHEMA, CREATE_EMPTY, MAJOR): + query = query.replace("{ ... on TypeResource", "{ __typename ... on TypeResource") + query = query.replace( + "{ id } }\n}", "{ id } ... on OperationInfo { messages { message } } }\n}" + ) + query = query.replace( + "{ name } }\n}", "{ name } ... on OperationInfo { messages { message } } }\n}" + ) + r = run(user, query, **variables) + assert denied(r), (who, r.errors, r.data) + assert Resource.objects.filter(id=world.pub_res.id, name="p.csv").exists() + + +@pytest.mark.parametrize("who", ["owner", "editor", "admin", "shared"]) +def test_people_with_edit_rights_can_rename_and_delete(world, who): + user = world.users[who] + r = run(user, RENAME, id=str(world.draft_res.id)) + assert r.errors is None, r.errors + assert r.data["updateFileResource"]["name"] == "renamed" + + r = run(user, DELETE, id=str(world.draft_res.id)) + assert r.errors is None and r.data["deleteFileResource"] is True + assert not Resource.objects.filter(id=world.draft_res.id).exists() + + +def test_shared_editor_loses_rights_once_published(world): + # Dataset-level roles only cover drafts, as for dataset edits. + r = run(world.users["shared"], DELETE, id=str(world.pub_res.id)) + assert denied(r) + + +def test_missing_id_is_refused_like_a_forbidden_one(world): + r = run(AnonymousUser(), DELETE, id=str(uuid.uuid4())) + assert denied(r) + assert "does not exist" not in str(r.errors) + + +LIST = "query ($ds: UUID!) { datasetResources(datasetId: $ds) { name } }" +ONE = "query ($id: UUID!) { resourceById(resourceId: $id) { name } }" + + +def test_published_resources_stay_public(world): + r = run(AnonymousUser(), LIST, ds=str(world.published.id)) + assert r.errors is None and r.data["datasetResources"] == [{"name": "p.csv"}] + r = run(AnonymousUser(), ONE, id=str(world.pub_res.id)) + assert r.errors is None and r.data["resourceById"]["name"] == "p.csv" + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +def test_draft_resources_are_hidden(world, who): + user = AnonymousUser() if who == "anonymous" else world.users[who] + r = run(user, LIST, ds=str(world.draft.id)) + assert r.errors is None and r.data["datasetResources"] == [] + r = run(user, ONE, id=str(world.draft_res.id)) + assert r.errors and "does not exist" in str(r.errors[0].message) + + +@pytest.mark.parametrize("who", ["owner", "editor", "viewer", "shared", "admin"]) +def test_draft_resources_visible_to_people_who_can_see_the_draft(world, who): + user = world.users[who] + r = run(user, LIST, ds=str(world.draft.id)) + assert r.errors is None and r.data["datasetResources"] == [{"name": "d.csv"}] + r = run(user, ONE, id=str(world.draft_res.id)) + assert r.errors is None and r.data["resourceById"]["name"] == "d.csv" + + +def test_all_resources_list_hides_drafts(world): + query = "{ resources { name } }" + r = run(AnonymousUser(), query) + assert r.errors is None and r.data["resources"] == [{"name": "p.csv"}] + r = run(world.users["outsider"], query) + assert r.data["resources"] == [{"name": "p.csv"}] + r = run(world.users["owner"], query) + assert sorted(x["name"] for x in r.data["resources"]) == ["d.csv", "p.csv"] From 2abfa6ad6f9aa7b56ccb28e091052f683fcd36b0 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Sun, 1 Nov 2026 00:46:31 +0530 Subject: [PATCH 29/55] feat(api): offer every license in licenses.csv as a DatasetLicense choice DatasetLicense had 5 values while the license contract lists 60. It now has one choice per CSV row, labelled with the CSV label. The 5 existing values keep their names so stored rows are untouched; the new ones are named after their SPDX code (CC-BY-NC-4.0 -> CC_BY_NC_4_0). Every CSV row now carries a dataspace_enum alt code, so the metadata export resolves a URI for each value. Migration 0056 only updates the field choices on Dataset and Publication (no SQL). Co-Authored-By: Claude Opus 5.5 --- api/migrations/0056_update_license_choices.py | 285 ++++++++++++++++++ .../metadata_export/contracts/licenses.csv | 110 +++---- api/utils/enums.py | 130 +++++++- tests/test_license_list.py | 123 ++++++++ 4 files changed, 588 insertions(+), 60 deletions(-) create mode 100644 api/migrations/0056_update_license_choices.py create mode 100644 tests/test_license_list.py diff --git a/api/migrations/0056_update_license_choices.py b/api/migrations/0056_update_license_choices.py new file mode 100644 index 00000000..bd87449d --- /dev/null +++ b/api/migrations/0056_update_license_choices.py @@ -0,0 +1,285 @@ +# Generated by Django 5.0.4 on 2026-10-31 18:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0055_event"), + ] + + operations = [ + migrations.AlterField( + model_name="dataset", + name="license", + field=models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License – India"), + ("CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License 1.0"), + ("CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal"), + ("CC_BY_1_0", "Creative Commons Attribution 1.0 Generic"), + ("CC_BY_2_0", "Creative Commons Attribution 2.0 Generic"), + ("CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia"), + ("CC_BY_2_5", "Creative Commons Attribution 2.5 Generic"), + ("CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia"), + ("CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria"), + ("CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany"), + ("CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO"), + ("CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands"), + ("CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States"), + ("CC_BY_3_0", "Creative Commons Attribution 3.0 Unported"), + ("CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic"), + ("CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic"), + ("CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic"), + ("CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany"), + ("CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported"), + ( + "CC_BY_ND_4_0", + "Creative Commons Attribution No Derivatives 4.0 International", + ), + ("CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic"), + ("CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic"), + ("CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic"), + ("CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany"), + ("CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO"), + ("CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported"), + ( + "CC_BY_NC_4_0", + "Creative Commons Attribution Non Commercial 4.0 International", + ), + ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ), + ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ), + ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ), + ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ), + ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ), + ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ), + ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ), + ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ), + ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ), + ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ), + ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ), + ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ), + ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ), + ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ), + ("CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic"), + ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ), + ("CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic"), + ("CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan"), + ("CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic"), + ("CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria"), + ("CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany"), + ("CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported"), + ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ), + ("CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO"), + ("CC_PDDC", "Creative Commons Public Domain Dedication and Certification"), + ("CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic"), + ("CC0_1_0", "Creative Commons Zero v1.0 Universal"), + ("PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + migrations.AlterField( + model_name="publication", + name="license", + field=models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License – India"), + ("CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License 1.0"), + ("CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal"), + ("CC_BY_1_0", "Creative Commons Attribution 1.0 Generic"), + ("CC_BY_2_0", "Creative Commons Attribution 2.0 Generic"), + ("CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia"), + ("CC_BY_2_5", "Creative Commons Attribution 2.5 Generic"), + ("CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia"), + ("CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria"), + ("CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany"), + ("CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO"), + ("CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands"), + ("CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States"), + ("CC_BY_3_0", "Creative Commons Attribution 3.0 Unported"), + ("CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic"), + ("CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic"), + ("CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic"), + ("CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany"), + ("CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported"), + ( + "CC_BY_ND_4_0", + "Creative Commons Attribution No Derivatives 4.0 International", + ), + ("CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic"), + ("CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic"), + ("CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic"), + ("CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany"), + ("CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO"), + ("CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported"), + ( + "CC_BY_NC_4_0", + "Creative Commons Attribution Non Commercial 4.0 International", + ), + ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ), + ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ), + ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ), + ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ), + ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ), + ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ), + ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ), + ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ), + ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ), + ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ), + ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ), + ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ), + ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ), + ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ), + ("CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic"), + ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ), + ("CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic"), + ("CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan"), + ("CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic"), + ("CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria"), + ("CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany"), + ("CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported"), + ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ), + ("CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO"), + ("CC_PDDC", "Creative Commons Public Domain Dedication and Certification"), + ("CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic"), + ("CC0_1_0", "Creative Commons Zero v1.0 Universal"), + ("PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + ] diff --git a/api/services/metadata_export/contracts/licenses.csv b/api/services/metadata_export/contracts/licenses.csv index cba256f4..85fd1a65 100644 --- a/api/services/metadata_export/contracts/licenses.csv +++ b/api/services/metadata_export/contracts/licenses.csv @@ -1,61 +1,61 @@ key,label,code,uri,is_open,visible_on_dataspace,alt_codes GODL-India,Government Open Data License – India,GODL-India,https://civicdataspace.in/id/licence/godl-india,True,yes,dataspace_enum=GOVERNMENT_OPEN_DATA_LICENSE -CC-PDM-1.0,Creative Commons Public Domain Mark 1.0 Universal,CC-PDM-1.0,https://spdx.org/licenses/CC-PDM-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-PDM-1.0.json -CC-BY-1.0,Creative Commons Attribution 1.0 Generic,CC-BY-1.0,https://spdx.org/licenses/CC-BY-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-1.0.json -CC-BY-2.0,Creative Commons Attribution 2.0 Generic,CC-BY-2.0,https://spdx.org/licenses/CC-BY-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.0.json -CC-BY-2.5-AU,Creative Commons Attribution 2.5 Australia,CC-BY-2.5-AU,https://spdx.org/licenses/CC-BY-2.5-AU,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.5-AU.json -CC-BY-2.5,Creative Commons Attribution 2.5 Generic,CC-BY-2.5,https://spdx.org/licenses/CC-BY-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.5.json -CC-BY-3.0-AU,Creative Commons Attribution 3.0 Australia,CC-BY-3.0-AU,https://spdx.org/licenses/CC-BY-3.0-AU,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AU.json -CC-BY-3.0-AT,Creative Commons Attribution 3.0 Austria,CC-BY-3.0-AT,https://spdx.org/licenses/CC-BY-3.0-AT,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AT.json -CC-BY-3.0-DE,Creative Commons Attribution 3.0 Germany,CC-BY-3.0-DE,https://spdx.org/licenses/CC-BY-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-DE.json -CC-BY-3.0-IGO,Creative Commons Attribution 3.0 IGO,CC-BY-3.0-IGO,https://spdx.org/licenses/CC-BY-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-IGO.json -CC-BY-3.0-NL,Creative Commons Attribution 3.0 Netherlands,CC-BY-3.0-NL,https://spdx.org/licenses/CC-BY-3.0-NL,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-NL.json -CC-BY-3.0-US,Creative Commons Attribution 3.0 United States,CC-BY-3.0-US,https://spdx.org/licenses/CC-BY-3.0-US,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-US.json -CC-BY-3.0,Creative Commons Attribution 3.0 Unported,CC-BY-3.0,https://spdx.org/licenses/CC-BY-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0.json +CC-PDM-1.0,Creative Commons Public Domain Mark 1.0 Universal,CC-PDM-1.0,https://spdx.org/licenses/CC-PDM-1.0,False,no,dataspace_enum=CC_PDM_1_0|spdx_detail=https://spdx.org/licenses/CC-PDM-1.0.json +CC-BY-1.0,Creative Commons Attribution 1.0 Generic,CC-BY-1.0,https://spdx.org/licenses/CC-BY-1.0,False,no,dataspace_enum=CC_BY_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-1.0.json +CC-BY-2.0,Creative Commons Attribution 2.0 Generic,CC-BY-2.0,https://spdx.org/licenses/CC-BY-2.0,False,no,dataspace_enum=CC_BY_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-2.0.json +CC-BY-2.5-AU,Creative Commons Attribution 2.5 Australia,CC-BY-2.5-AU,https://spdx.org/licenses/CC-BY-2.5-AU,False,no,dataspace_enum=CC_BY_2_5_AU|spdx_detail=https://spdx.org/licenses/CC-BY-2.5-AU.json +CC-BY-2.5,Creative Commons Attribution 2.5 Generic,CC-BY-2.5,https://spdx.org/licenses/CC-BY-2.5,False,no,dataspace_enum=CC_BY_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-2.5.json +CC-BY-3.0-AU,Creative Commons Attribution 3.0 Australia,CC-BY-3.0-AU,https://spdx.org/licenses/CC-BY-3.0-AU,False,no,dataspace_enum=CC_BY_3_0_AU|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AU.json +CC-BY-3.0-AT,Creative Commons Attribution 3.0 Austria,CC-BY-3.0-AT,https://spdx.org/licenses/CC-BY-3.0-AT,False,no,dataspace_enum=CC_BY_3_0_AT|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AT.json +CC-BY-3.0-DE,Creative Commons Attribution 3.0 Germany,CC-BY-3.0-DE,https://spdx.org/licenses/CC-BY-3.0-DE,False,no,dataspace_enum=CC_BY_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-DE.json +CC-BY-3.0-IGO,Creative Commons Attribution 3.0 IGO,CC-BY-3.0-IGO,https://spdx.org/licenses/CC-BY-3.0-IGO,False,no,dataspace_enum=CC_BY_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-IGO.json +CC-BY-3.0-NL,Creative Commons Attribution 3.0 Netherlands,CC-BY-3.0-NL,https://spdx.org/licenses/CC-BY-3.0-NL,False,no,dataspace_enum=CC_BY_3_0_NL|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-NL.json +CC-BY-3.0-US,Creative Commons Attribution 3.0 United States,CC-BY-3.0-US,https://spdx.org/licenses/CC-BY-3.0-US,False,no,dataspace_enum=CC_BY_3_0_US|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-US.json +CC-BY-3.0,Creative Commons Attribution 3.0 Unported,CC-BY-3.0,https://spdx.org/licenses/CC-BY-3.0,False,no,dataspace_enum=CC_BY_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-3.0.json CC-BY-4.0,Creative Commons Attribution 4.0,CC-BY-4.0,http://creativecommons.org/licenses/by/4.0/,True,yes,authority_label=Creative Commons Attribution 4.0 International|dataspace_enum=CC_BY_4_0_ATTRIBUTION|spdx_detail=https://spdx.org/licenses/CC-BY-4.0.json -CC-BY-ND-1.0,Creative Commons Attribution No Derivatives 1.0 Generic,CC-BY-ND-1.0,https://spdx.org/licenses/CC-BY-ND-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-1.0.json -CC-BY-ND-2.0,Creative Commons Attribution No Derivatives 2.0 Generic,CC-BY-ND-2.0,https://spdx.org/licenses/CC-BY-ND-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.0.json -CC-BY-ND-2.5,Creative Commons Attribution No Derivatives 2.5 Generic,CC-BY-ND-2.5,https://spdx.org/licenses/CC-BY-ND-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.5.json -CC-BY-ND-3.0-DE,Creative Commons Attribution No Derivatives 3.0 Germany,CC-BY-ND-3.0-DE,https://spdx.org/licenses/CC-BY-ND-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0-DE.json -CC-BY-ND-3.0,Creative Commons Attribution No Derivatives 3.0 Unported,CC-BY-ND-3.0,https://spdx.org/licenses/CC-BY-ND-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0.json -CC-BY-ND-4.0,Creative Commons Attribution No Derivatives 4.0 International,CC-BY-ND-4.0,https://spdx.org/licenses/CC-BY-ND-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-4.0.json -CC-BY-NC-1.0,Creative Commons Attribution Non Commercial 1.0 Generic,CC-BY-NC-1.0,https://spdx.org/licenses/CC-BY-NC-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-1.0.json -CC-BY-NC-2.0,Creative Commons Attribution Non Commercial 2.0 Generic,CC-BY-NC-2.0,https://spdx.org/licenses/CC-BY-NC-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.0.json -CC-BY-NC-2.5,Creative Commons Attribution Non Commercial 2.5 Generic,CC-BY-NC-2.5,https://spdx.org/licenses/CC-BY-NC-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.5.json -CC-BY-NC-3.0-DE,Creative Commons Attribution Non Commercial 3.0 Germany,CC-BY-NC-3.0-DE,https://spdx.org/licenses/CC-BY-NC-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-DE.json -CC-BY-NC-3.0-IGO,Creative Commons Attribution Non Commercial 3.0 IGO,CC-BY-NC-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-IGO.json -CC-BY-NC-3.0,Creative Commons Attribution Non Commercial 3.0 Unported,CC-BY-NC-3.0,https://spdx.org/licenses/CC-BY-NC-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0.json -CC-BY-NC-4.0,Creative Commons Attribution Non Commercial 4.0 International,CC-BY-NC-4.0,http://creativecommons.org/licenses/by-nc/4.0/,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-4.0.json -CC-BY-NC-ND-1.0,Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic,CC-BY-NC-ND-1.0,https://spdx.org/licenses/CC-BY-NC-ND-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-1.0.json -CC-BY-NC-ND-2.0,Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic,CC-BY-NC-ND-2.0,https://spdx.org/licenses/CC-BY-NC-ND-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.0.json -CC-BY-NC-ND-2.5,Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic,CC-BY-NC-ND-2.5,https://spdx.org/licenses/CC-BY-NC-ND-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.5.json -CC-BY-NC-ND-3.0-DE,Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany,CC-BY-NC-ND-3.0-DE,https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json -CC-BY-NC-ND-3.0-IGO,Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO,CC-BY-NC-ND-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json -CC-BY-NC-ND-3.0,Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported,CC-BY-NC-ND-3.0,https://spdx.org/licenses/CC-BY-NC-ND-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0.json -CC-BY-NC-ND-4.0,Creative Commons Attribution Non Commercial No Derivatives 4.0 International,CC-BY-NC-ND-4.0,https://spdx.org/licenses/CC-BY-NC-ND-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-4.0.json -CC-BY-NC-SA-1.0,Creative Commons Attribution Non Commercial Share Alike 1.0 Generic,CC-BY-NC-SA-1.0,https://spdx.org/licenses/CC-BY-NC-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-1.0.json -CC-BY-NC-SA-2.0-UK,Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales,CC-BY-NC-SA-2.0-UK,https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json -CC-BY-NC-SA-2.0,Creative Commons Attribution Non Commercial Share Alike 2.0 Generic,CC-BY-NC-SA-2.0,https://spdx.org/licenses/CC-BY-NC-SA-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0.json -CC-BY-NC-SA-2.0-DE,Creative Commons Attribution Non Commercial Share Alike 2.0 Germany,CC-BY-NC-SA-2.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.json -CC-BY-NC-SA-2.5,Creative Commons Attribution Non Commercial Share Alike 2.5 Generic,CC-BY-NC-SA-2.5,https://spdx.org/licenses/CC-BY-NC-SA-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.5.json -CC-BY-NC-SA-3.0-DE,Creative Commons Attribution Non Commercial Share Alike 3.0 Germany,CC-BY-NC-SA-3.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json -CC-BY-NC-SA-3.0-IGO,Creative Commons Attribution Non Commercial Share Alike 3.0 IGO,CC-BY-NC-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json -CC-BY-NC-SA-3.0,Creative Commons Attribution Non Commercial Share Alike 3.0 Unported,CC-BY-NC-SA-3.0,https://spdx.org/licenses/CC-BY-NC-SA-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0.json -CC-BY-NC-SA-4.0,Creative Commons Attribution Non Commercial Share Alike 4.0 International,CC-BY-NC-SA-4.0,https://spdx.org/licenses/CC-BY-NC-SA-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-4.0.json -CC-BY-SA-1.0,Creative Commons Attribution Share Alike 1.0 Generic,CC-BY-SA-1.0,https://spdx.org/licenses/CC-BY-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-1.0.json -CC-BY-SA-2.0-UK,Creative Commons Attribution Share Alike 2.0 England and Wales,CC-BY-SA-2.0-UK,https://spdx.org/licenses/CC-BY-SA-2.0-UK,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0-UK.json -CC-BY-SA-2.0,Creative Commons Attribution Share Alike 2.0 Generic,CC-BY-SA-2.0,https://spdx.org/licenses/CC-BY-SA-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0.json -CC-BY-SA-2.1-JP,Creative Commons Attribution Share Alike 2.1 Japan,CC-BY-SA-2.1-JP,https://spdx.org/licenses/CC-BY-SA-2.1-JP,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.1-JP.json -CC-BY-SA-2.5,Creative Commons Attribution Share Alike 2.5 Generic,CC-BY-SA-2.5,https://spdx.org/licenses/CC-BY-SA-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.5.json -CC-BY-SA-3.0-AT,Creative Commons Attribution Share Alike 3.0 Austria,CC-BY-SA-3.0-AT,https://spdx.org/licenses/CC-BY-SA-3.0-AT,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-AT.json -CC-BY-SA-3.0-DE,Creative Commons Attribution Share Alike 3.0 Germany,CC-BY-SA-3.0-DE,https://spdx.org/licenses/CC-BY-SA-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-DE.json -CC-BY-SA-3.0,Creative Commons Attribution Share Alike 3.0 Unported,CC-BY-SA-3.0,https://spdx.org/licenses/CC-BY-SA-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0.json -CC-BY-NC-SA-2.0-FR,Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France,CC-BY-NC-SA-2.0-FR,https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json -CC-BY-SA-3.0-IGO,Creative Commons Attribution-ShareAlike 3.0 IGO,CC-BY-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-SA-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-IGO.json +CC-BY-ND-1.0,Creative Commons Attribution No Derivatives 1.0 Generic,CC-BY-ND-1.0,https://spdx.org/licenses/CC-BY-ND-1.0,False,no,dataspace_enum=CC_BY_ND_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-1.0.json +CC-BY-ND-2.0,Creative Commons Attribution No Derivatives 2.0 Generic,CC-BY-ND-2.0,https://spdx.org/licenses/CC-BY-ND-2.0,False,no,dataspace_enum=CC_BY_ND_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.0.json +CC-BY-ND-2.5,Creative Commons Attribution No Derivatives 2.5 Generic,CC-BY-ND-2.5,https://spdx.org/licenses/CC-BY-ND-2.5,False,no,dataspace_enum=CC_BY_ND_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.5.json +CC-BY-ND-3.0-DE,Creative Commons Attribution No Derivatives 3.0 Germany,CC-BY-ND-3.0-DE,https://spdx.org/licenses/CC-BY-ND-3.0-DE,False,no,dataspace_enum=CC_BY_ND_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0-DE.json +CC-BY-ND-3.0,Creative Commons Attribution No Derivatives 3.0 Unported,CC-BY-ND-3.0,https://spdx.org/licenses/CC-BY-ND-3.0,False,no,dataspace_enum=CC_BY_ND_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0.json +CC-BY-ND-4.0,Creative Commons Attribution No Derivatives 4.0 International,CC-BY-ND-4.0,https://spdx.org/licenses/CC-BY-ND-4.0,False,no,dataspace_enum=CC_BY_ND_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-4.0.json +CC-BY-NC-1.0,Creative Commons Attribution Non Commercial 1.0 Generic,CC-BY-NC-1.0,https://spdx.org/licenses/CC-BY-NC-1.0,False,no,dataspace_enum=CC_BY_NC_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-1.0.json +CC-BY-NC-2.0,Creative Commons Attribution Non Commercial 2.0 Generic,CC-BY-NC-2.0,https://spdx.org/licenses/CC-BY-NC-2.0,False,no,dataspace_enum=CC_BY_NC_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.0.json +CC-BY-NC-2.5,Creative Commons Attribution Non Commercial 2.5 Generic,CC-BY-NC-2.5,https://spdx.org/licenses/CC-BY-NC-2.5,False,no,dataspace_enum=CC_BY_NC_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.5.json +CC-BY-NC-3.0-DE,Creative Commons Attribution Non Commercial 3.0 Germany,CC-BY-NC-3.0-DE,https://spdx.org/licenses/CC-BY-NC-3.0-DE,False,no,dataspace_enum=CC_BY_NC_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-DE.json +CC-BY-NC-3.0-IGO,Creative Commons Attribution Non Commercial 3.0 IGO,CC-BY-NC-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-IGO.json +CC-BY-NC-3.0,Creative Commons Attribution Non Commercial 3.0 Unported,CC-BY-NC-3.0,https://spdx.org/licenses/CC-BY-NC-3.0,False,no,dataspace_enum=CC_BY_NC_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0.json +CC-BY-NC-4.0,Creative Commons Attribution Non Commercial 4.0 International,CC-BY-NC-4.0,http://creativecommons.org/licenses/by-nc/4.0/,False,no,dataspace_enum=CC_BY_NC_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-4.0.json +CC-BY-NC-ND-1.0,Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic,CC-BY-NC-ND-1.0,https://spdx.org/licenses/CC-BY-NC-ND-1.0,False,no,dataspace_enum=CC_BY_NC_ND_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-1.0.json +CC-BY-NC-ND-2.0,Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic,CC-BY-NC-ND-2.0,https://spdx.org/licenses/CC-BY-NC-ND-2.0,False,no,dataspace_enum=CC_BY_NC_ND_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.0.json +CC-BY-NC-ND-2.5,Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic,CC-BY-NC-ND-2.5,https://spdx.org/licenses/CC-BY-NC-ND-2.5,False,no,dataspace_enum=CC_BY_NC_ND_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.5.json +CC-BY-NC-ND-3.0-DE,Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany,CC-BY-NC-ND-3.0-DE,https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE,False,no,dataspace_enum=CC_BY_NC_ND_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json +CC-BY-NC-ND-3.0-IGO,Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO,CC-BY-NC-ND-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_ND_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json +CC-BY-NC-ND-3.0,Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported,CC-BY-NC-ND-3.0,https://spdx.org/licenses/CC-BY-NC-ND-3.0,False,no,dataspace_enum=CC_BY_NC_ND_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0.json +CC-BY-NC-ND-4.0,Creative Commons Attribution Non Commercial No Derivatives 4.0 International,CC-BY-NC-ND-4.0,https://spdx.org/licenses/CC-BY-NC-ND-4.0,False,no,dataspace_enum=CC_BY_NC_ND_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-4.0.json +CC-BY-NC-SA-1.0,Creative Commons Attribution Non Commercial Share Alike 1.0 Generic,CC-BY-NC-SA-1.0,https://spdx.org/licenses/CC-BY-NC-SA-1.0,False,no,dataspace_enum=CC_BY_NC_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-1.0.json +CC-BY-NC-SA-2.0-UK,Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales,CC-BY-NC-SA-2.0-UK,https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK,False,no,dataspace_enum=CC_BY_NC_SA_2_0_UK|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json +CC-BY-NC-SA-2.0,Creative Commons Attribution Non Commercial Share Alike 2.0 Generic,CC-BY-NC-SA-2.0,https://spdx.org/licenses/CC-BY-NC-SA-2.0,False,no,dataspace_enum=CC_BY_NC_SA_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0.json +CC-BY-NC-SA-2.0-DE,Creative Commons Attribution Non Commercial Share Alike 2.0 Germany,CC-BY-NC-SA-2.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE,False,no,dataspace_enum=CC_BY_NC_SA_2_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.json +CC-BY-NC-SA-2.5,Creative Commons Attribution Non Commercial Share Alike 2.5 Generic,CC-BY-NC-SA-2.5,https://spdx.org/licenses/CC-BY-NC-SA-2.5,False,no,dataspace_enum=CC_BY_NC_SA_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.5.json +CC-BY-NC-SA-3.0-DE,Creative Commons Attribution Non Commercial Share Alike 3.0 Germany,CC-BY-NC-SA-3.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE,False,no,dataspace_enum=CC_BY_NC_SA_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json +CC-BY-NC-SA-3.0-IGO,Creative Commons Attribution Non Commercial Share Alike 3.0 IGO,CC-BY-NC-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_SA_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json +CC-BY-NC-SA-3.0,Creative Commons Attribution Non Commercial Share Alike 3.0 Unported,CC-BY-NC-SA-3.0,https://spdx.org/licenses/CC-BY-NC-SA-3.0,False,no,dataspace_enum=CC_BY_NC_SA_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0.json +CC-BY-NC-SA-4.0,Creative Commons Attribution Non Commercial Share Alike 4.0 International,CC-BY-NC-SA-4.0,https://spdx.org/licenses/CC-BY-NC-SA-4.0,False,no,dataspace_enum=CC_BY_NC_SA_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-4.0.json +CC-BY-SA-1.0,Creative Commons Attribution Share Alike 1.0 Generic,CC-BY-SA-1.0,https://spdx.org/licenses/CC-BY-SA-1.0,False,no,dataspace_enum=CC_BY_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-1.0.json +CC-BY-SA-2.0-UK,Creative Commons Attribution Share Alike 2.0 England and Wales,CC-BY-SA-2.0-UK,https://spdx.org/licenses/CC-BY-SA-2.0-UK,False,no,dataspace_enum=CC_BY_SA_2_0_UK|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0-UK.json +CC-BY-SA-2.0,Creative Commons Attribution Share Alike 2.0 Generic,CC-BY-SA-2.0,https://spdx.org/licenses/CC-BY-SA-2.0,False,no,dataspace_enum=CC_BY_SA_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0.json +CC-BY-SA-2.1-JP,Creative Commons Attribution Share Alike 2.1 Japan,CC-BY-SA-2.1-JP,https://spdx.org/licenses/CC-BY-SA-2.1-JP,False,no,dataspace_enum=CC_BY_SA_2_1_JP|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.1-JP.json +CC-BY-SA-2.5,Creative Commons Attribution Share Alike 2.5 Generic,CC-BY-SA-2.5,https://spdx.org/licenses/CC-BY-SA-2.5,False,no,dataspace_enum=CC_BY_SA_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.5.json +CC-BY-SA-3.0-AT,Creative Commons Attribution Share Alike 3.0 Austria,CC-BY-SA-3.0-AT,https://spdx.org/licenses/CC-BY-SA-3.0-AT,False,no,dataspace_enum=CC_BY_SA_3_0_AT|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-AT.json +CC-BY-SA-3.0-DE,Creative Commons Attribution Share Alike 3.0 Germany,CC-BY-SA-3.0-DE,https://spdx.org/licenses/CC-BY-SA-3.0-DE,False,no,dataspace_enum=CC_BY_SA_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-DE.json +CC-BY-SA-3.0,Creative Commons Attribution Share Alike 3.0 Unported,CC-BY-SA-3.0,https://spdx.org/licenses/CC-BY-SA-3.0,False,no,dataspace_enum=CC_BY_SA_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0.json +CC-BY-NC-SA-2.0-FR,Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France,CC-BY-NC-SA-2.0-FR,https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR,False,no,dataspace_enum=CC_BY_NC_SA_2_0_FR|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json +CC-BY-SA-3.0-IGO,Creative Commons Attribution-ShareAlike 3.0 IGO,CC-BY-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-SA-3.0-IGO,False,no,dataspace_enum=CC_BY_SA_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-IGO.json CC-BY-SA-4.0,Creative Commons Attribution-ShareAlike 4.0,CC-BY-SA-4.0,http://creativecommons.org/licenses/by-sa/4.0/,True,yes,authority_label=Creative Commons Attribution Share Alike 4.0 International|dataspace_enum=CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE|spdx_detail=https://spdx.org/licenses/CC-BY-SA-4.0.json -CC-PDDC,Creative Commons Public Domain Dedication and Certification,CC-PDDC,https://spdx.org/licenses/CC-PDDC,False,no,spdx_detail=https://spdx.org/licenses/CC-PDDC.json -CC-SA-1.0,Creative Commons Share Alike 1.0 Generic,CC-SA-1.0,https://spdx.org/licenses/CC-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-SA-1.0.json -CC0-1.0,Creative Commons Zero v1.0 Universal,CC0-1.0,https://creativecommons.org/publicdomain/zero/1.0/,True,yes,spdx_detail=https://spdx.org/licenses/CC0-1.0.json +CC-PDDC,Creative Commons Public Domain Dedication and Certification,CC-PDDC,https://spdx.org/licenses/CC-PDDC,False,no,dataspace_enum=CC_PDDC|spdx_detail=https://spdx.org/licenses/CC-PDDC.json +CC-SA-1.0,Creative Commons Share Alike 1.0 Generic,CC-SA-1.0,https://spdx.org/licenses/CC-SA-1.0,False,no,dataspace_enum=CC_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-SA-1.0.json +CC0-1.0,Creative Commons Zero v1.0 Universal,CC0-1.0,https://creativecommons.org/publicdomain/zero/1.0/,True,yes,dataspace_enum=CC0_1_0|spdx_detail=https://spdx.org/licenses/CC0-1.0.json ODC-By-1.0,Open Data Commons Attribution License 1.0,ODC-By-1.0,https://opendatacommons.org/licenses/by/1-0/,True,yes,authority_label=Open Data Commons Attribution License v1.0|dataspace_enum=OPEN_DATA_COMMONS_BY_ATTRIBUTION|spdx_detail=https://spdx.org/licenses/ODC-By-1.0.json -PDDL-1.0,Open Data Commons Public Domain Dedication & License 1.0,PDDL-1.0,https://opendatacommons.org/licenses/pddl/1-0/,True,yes,spdx_detail=https://spdx.org/licenses/PDDL-1.0.json +PDDL-1.0,Open Data Commons Public Domain Dedication & License 1.0,PDDL-1.0,https://opendatacommons.org/licenses/pddl/1-0/,True,yes,dataspace_enum=PDDL_1_0|spdx_detail=https://spdx.org/licenses/PDDL-1.0.json ODbL-1.0,Open Database License 1.0,ODbL-1.0,https://opendatacommons.org/licenses/odbl/1-0/,True,yes,authority_label=Open Data Commons Open Database License v1.0|dataspace_enum=OPEN_DATABASE_LICENSE|spdx_detail=https://spdx.org/licenses/ODbL-1.0.json diff --git a/api/utils/enums.py b/api/utils/enums.py index 5b8874de..686153fe 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -179,11 +179,131 @@ class DatasetAccessType(models.TextChoices): class DatasetLicense(models.TextChoices): - GOVERNMENT_OPEN_DATA_LICENSE = "GOVERNMENT_OPEN_DATA_LICENSE" - CC_BY_4_0_ATTRIBUTION = "CC_BY_4_0_ATTRIBUTION" - CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE = "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE" - OPEN_DATA_COMMONS_BY_ATTRIBUTION = "OPEN_DATA_COMMONS_BY_ATTRIBUTION" - OPEN_DATABASE_LICENSE = "OPEN_DATABASE_LICENSE" + """One value per row of contracts/licenses.csv (matched by its dataspace_enum alt code).""" + + GOVERNMENT_OPEN_DATA_LICENSE = ( + "GOVERNMENT_OPEN_DATA_LICENSE", + "Government Open Data License – India", + ) + CC_BY_4_0_ATTRIBUTION = "CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0" + CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE = ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ) + OPEN_DATA_COMMONS_BY_ATTRIBUTION = ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ) + OPEN_DATABASE_LICENSE = "OPEN_DATABASE_LICENSE", "Open Database License 1.0" + CC_PDM_1_0 = "CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal" + CC_BY_1_0 = "CC_BY_1_0", "Creative Commons Attribution 1.0 Generic" + CC_BY_2_0 = "CC_BY_2_0", "Creative Commons Attribution 2.0 Generic" + CC_BY_2_5_AU = "CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia" + CC_BY_2_5 = "CC_BY_2_5", "Creative Commons Attribution 2.5 Generic" + CC_BY_3_0_AU = "CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia" + CC_BY_3_0_AT = "CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria" + CC_BY_3_0_DE = "CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany" + CC_BY_3_0_IGO = "CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO" + CC_BY_3_0_NL = "CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands" + CC_BY_3_0_US = "CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States" + CC_BY_3_0 = "CC_BY_3_0", "Creative Commons Attribution 3.0 Unported" + CC_BY_ND_1_0 = "CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic" + CC_BY_ND_2_0 = "CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic" + CC_BY_ND_2_5 = "CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic" + CC_BY_ND_3_0_DE = "CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany" + CC_BY_ND_3_0 = "CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported" + CC_BY_ND_4_0 = "CC_BY_ND_4_0", "Creative Commons Attribution No Derivatives 4.0 International" + CC_BY_NC_1_0 = "CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic" + CC_BY_NC_2_0 = "CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic" + CC_BY_NC_2_5 = "CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic" + CC_BY_NC_3_0_DE = "CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany" + CC_BY_NC_3_0_IGO = "CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO" + CC_BY_NC_3_0 = "CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported" + CC_BY_NC_4_0 = "CC_BY_NC_4_0", "Creative Commons Attribution Non Commercial 4.0 International" + CC_BY_NC_ND_1_0 = ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ) + CC_BY_NC_ND_2_0 = ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ) + CC_BY_NC_ND_2_5 = ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ) + CC_BY_NC_ND_3_0_DE = ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ) + CC_BY_NC_ND_3_0_IGO = ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ) + CC_BY_NC_ND_3_0 = ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ) + CC_BY_NC_ND_4_0 = ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ) + CC_BY_NC_SA_1_0 = ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ) + CC_BY_NC_SA_2_0_UK = ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ) + CC_BY_NC_SA_2_0 = ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ) + CC_BY_NC_SA_2_0_DE = ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ) + CC_BY_NC_SA_2_5 = ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ) + CC_BY_NC_SA_3_0_DE = ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ) + CC_BY_NC_SA_3_0_IGO = ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ) + CC_BY_NC_SA_3_0 = ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ) + CC_BY_NC_SA_4_0 = ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ) + CC_BY_SA_1_0 = "CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic" + CC_BY_SA_2_0_UK = ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ) + CC_BY_SA_2_0 = "CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic" + CC_BY_SA_2_1_JP = "CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan" + CC_BY_SA_2_5 = "CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic" + CC_BY_SA_3_0_AT = "CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria" + CC_BY_SA_3_0_DE = "CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany" + CC_BY_SA_3_0 = "CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported" + CC_BY_NC_SA_2_0_FR = ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ) + CC_BY_SA_3_0_IGO = "CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO" + CC_PDDC = "CC_PDDC", "Creative Commons Public Domain Dedication and Certification" + CC_SA_1_0 = "CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic" + CC0_1_0 = "CC0_1_0", "Creative Commons Zero v1.0 Universal" + PDDL_1_0 = "PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0" class UseCaseStatus(models.TextChoices): diff --git a/tests/test_license_list.py b/tests/test_license_list.py new file mode 100644 index 00000000..2d3136c7 --- /dev/null +++ b/tests/test_license_list.py @@ -0,0 +1,123 @@ +"""DatasetLicense offers one choice per row of contracts/licenses.csv. + +The CSV is the source of truth: its ``dataspace_enum`` alt code names the enum +value, its label is the choice label, and its URI is what the metadata export +emits for that value. +""" + +import csv +import re +from pathlib import Path + +import pytest +from django.core.exceptions import ValidationError + +from api.models import Dataset, Publication +from api.schema.schema import schema +from api.services.metadata_export import vocabularies as vocab +from api.services.metadata_export.exporter import export_dataset +from api.services.publication_service import validate_draft_inputs +from api.utils.enums import DatasetLicense +from tests.schema.test_publication_schema import ( # noqa: F401 (fixtures) + CREATE, + _no_activity_recording, + ctx, + geography, + org_a, + org_a_admin, + resource_type, + roles, + run, + sector, + valid_create_vars, +) + +LICENSES_CSV = ( + Path(__file__).resolve().parent.parent / "api/services/metadata_export/contracts/licenses.csv" +) + + +def _rows(): + with open(LICENSES_CSV, encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +ROWS = _rows() + + +def _enum_value(row): + match = re.search(r"dataspace_enum=([A-Z0-9_]+)", row["alt_codes"]) + assert match, f"{row['key']} has no dataspace_enum alt code" + return match.group(1) + + +class TestEnumMatchesCsv: + def test_one_choice_per_row(self): + assert sorted(DatasetLicense.values) == sorted(_enum_value(r) for r in ROWS) + + def test_labels_come_from_the_csv(self): + labels = dict(DatasetLicense.choices) + for row in ROWS: + assert labels[_enum_value(row)] == " ".join(row["label"].split()), row["key"] + + def test_existing_values_are_unchanged(self): + # Stored rows use these; renaming one would orphan data. + for value in ( + "GOVERNMENT_OPEN_DATA_LICENSE", + "CC_BY_4_0_ATTRIBUTION", + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "OPEN_DATABASE_LICENSE", + ): + assert value in DatasetLicense.values + + def test_values_fit_the_column(self): + field = Dataset._meta.get_field("license") + assert max(len(v) for v in DatasetLicense.values) <= field.max_length + + @pytest.mark.parametrize("row", ROWS, ids=lambda r: r["key"]) + def test_every_value_resolves_to_its_uri(self, row): + assert vocab.license_uri(_enum_value(row)) == row["uri"] + + +@pytest.mark.django_db +class TestStorageAndExport: + @pytest.mark.parametrize("value", ["CC_BY_NC_ND_4_0", "CC0_1_0", "CC_BY_4_0_ATTRIBUTION"]) + def test_dataset_exports_the_license_uri(self, value, org_a): # noqa: F811 + dataset = Dataset.objects.create(title="Rainfall", organization=org_a, license=value) + body, _, _, _ = export_dataset(dataset, "dcat", "jsonld") + assert vocab.license_uri(value) in body + + def test_value_outside_the_list_is_rejected(self, org_a): # noqa: F811 + dataset = Dataset(title="Rainfall", organization=org_a, license="MIT") + with pytest.raises(ValidationError) as exc: + dataset.full_clean(exclude=["slug", "user", "dataspace", "description"]) + assert "license" in exc.value.message_dict + + +class TestGraphQL: + def test_enum_exposes_every_value(self): + result = schema.execute_sync('{ __type(name: "DatasetLicense") { enumValues { name } } }') + assert result.errors is None + names = {v["name"] for v in result.data["__type"]["enumValues"]} + assert names == set(DatasetLicense.values) + + @pytest.mark.django_db + @pytest.mark.parametrize("value", ["CC_BY_NC_4_0", "PDDL_1_0"]) + def test_create_publication_accepts_new_values( + self, value, org_a_admin, org_a, resource_type, sector, geography # noqa: F811 + ): + result = run( + CREATE, + ctx(org_a_admin, org_a), + valid_create_vars(resource_type, sector, geography, license=value), + ) + assert result.errors is None + payload = result.data["createPublication"] + assert payload["success"] is True + assert Publication.objects.get(id=payload["data"]["id"]).license == value + + def test_service_validation_uses_the_full_list(self): + validate_draft_inputs(license_value="CC_BY_SA_3_0_IGO") + with pytest.raises(ValidationError): + validate_draft_inputs(license_value="NOT_A_LICENSE") From 19f38f27634376f3ebfefb9b2bef837b3b821421 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Sun, 1 Nov 2026 00:46:31 +0530 Subject: [PATCH 30/55] fix(api): map SPDX license ids from imported datasets instead of defaulting to CC-BY map_license only knew a handful of aliases, so any other platform license (cc-by-nc-4.0, CC0, PDDL, ...) was stored as CC-BY 4.0 and exported as such. SPDX ids now match DatasetLicense directly, a few platform names (Kaggle's "CC0: Public Domain", PDDL, GODL) are added as aliases, and only unknown strings still fall back to CC-BY 4.0. Co-Authored-By: Claude Opus 5.5 --- api/services/platform_importers/base.py | 20 +++++-- tests/test_platform_license_mapping.py | 70 +++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/test_platform_license_mapping.py diff --git a/api/services/platform_importers/base.py b/api/services/platform_importers/base.py index 61518bfb..0b15f8ff 100644 --- a/api/services/platform_importers/base.py +++ b/api/services/platform_importers/base.py @@ -93,8 +93,10 @@ def mapped_license(self) -> str: # --------------------------------------------------------------------------- # # Helpers shared by importers # --------------------------------------------------------------------------- # -# Platform license strings (lower-cased) -> DatasetLicense. Anything not listed -# falls back to CC-BY 4.0 and the raw string is kept on DatasetSource. +# Platform license strings (lower-cased) -> DatasetLicense, for names that are +# not an SPDX id. SPDX ids ("CC-BY-NC-4.0") match the enum directly in +# map_license. Anything else falls back to CC-BY 4.0 and the raw string is kept +# on DatasetSource. LICENSE_ALIASES: Dict[str, str] = { "cc-by-4.0": DatasetLicense.CC_BY_4_0_ATTRIBUTION, "cc-by": DatasetLicense.CC_BY_4_0_ATTRIBUTION, @@ -113,13 +115,25 @@ def mapped_license(self) -> str: "odc-odbl": DatasetLicense.OPEN_DATABASE_LICENSE, "database: open database, contents: database contents": DatasetLicense.OPEN_DATABASE_LICENSE, "database: open database, contents: © original authors": DatasetLicense.OPEN_DATABASE_LICENSE, + "cc0": DatasetLicense.CC0_1_0, + "cc0: public domain": DatasetLicense.CC0_1_0, + "pddl": DatasetLicense.PDDL_1_0, + "odc-pddl": DatasetLicense.PDDL_1_0, + "odc public domain dedication and licence (pddl)": DatasetLicense.PDDL_1_0, + "godl": DatasetLicense.GOVERNMENT_OPEN_DATA_LICENSE, + "godl-india": DatasetLicense.GOVERNMENT_OPEN_DATA_LICENSE, } def map_license(raw: str) -> str: """Map a platform license string onto DatasetLicense (default CC-BY-4.0).""" key = (raw or "").strip().lower() - return LICENSE_ALIASES.get(key, DatasetLicense.CC_BY_4_0_ATTRIBUTION) + if key in LICENSE_ALIASES: + return LICENSE_ALIASES[key] + spdx = re.sub(r"[^A-Z0-9]+", "_", key.upper()).strip("_") + if spdx in DatasetLicense.values: + return spdx + return DatasetLicense.CC_BY_4_0_ATTRIBUTION # Hard limits: the platforms' own maxima are well under these, so anything diff --git a/tests/test_platform_license_mapping.py b/tests/test_platform_license_mapping.py new file mode 100644 index 00000000..2338ab4c --- /dev/null +++ b/tests/test_platform_license_mapping.py @@ -0,0 +1,70 @@ +"""Imported datasets keep the license the platform reported. + +SPDX ids map straight onto DatasetLicense; a few platform-specific names go +through LICENSE_ALIASES; anything unknown falls back to CC-BY 4.0. +""" + +import csv +import re +from pathlib import Path + +import pytest + +from api.services.platform_importers.base import map_license +from api.services.platform_importers.github import GitHubImporter +from api.services.platform_importers.huggingface import HuggingFaceImporter +from api.services.platform_importers.kaggle import KaggleImporter + +LICENSES_CSV = ( + Path(__file__).resolve().parent.parent / "api/services/metadata_export/contracts/licenses.csv" +) + +with open(LICENSES_CSV, encoding="utf-8", newline="") as fh: + ROWS = list(csv.DictReader(fh)) + + +def _enum_value(row): + return re.search(r"dataspace_enum=([A-Z0-9_]+)", row["alt_codes"]).group(1) + + +@pytest.mark.parametrize("row", ROWS, ids=lambda r: r["key"]) +def test_spdx_code_maps_to_its_value(row): + assert map_license(row["code"]) == _enum_value(row) + assert map_license(row["code"].lower()) == _enum_value(row) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("CC0: Public Domain", "CC0_1_0"), + ("CC BY-NC-SA 4.0", "CC_BY_NC_SA_4_0"), + ("Attribution 4.0 International (CC BY 4.0)", "CC_BY_4_0_ATTRIBUTION"), + ("ODbL", "OPEN_DATABASE_LICENSE"), + ("GODL-India", "GOVERNMENT_OPEN_DATA_LICENSE"), + ("MIT", "CC_BY_4_0_ATTRIBUTION"), + ("", "CC_BY_4_0_ATTRIBUTION"), + ], +) +def test_platform_names(raw, expected): + assert map_license(raw) == expected + + +def test_github_spdx_id(): + assert map_license(GitHubImporter._license({"spdx_id": "CC-BY-NC-4.0"})) == "CC_BY_NC_4_0" + assert map_license(GitHubImporter._license({"spdx_id": "NOASSERTION"})) == ( + "CC_BY_4_0_ATTRIBUTION" + ) + + +def test_huggingface_card_and_tag(): + assert map_license(HuggingFaceImporter._license({"license": "cc-by-nc-sa-4.0"}, [])) == ( + "CC_BY_NC_SA_4_0" + ) + assert map_license(HuggingFaceImporter._license({}, ["license:pddl"])) == "PDDL_1_0" + + +def test_kaggle_license_name(): + assert map_license(KaggleImporter._license({"licenses": [{"name": "CC0-1.0"}]})) == "CC0_1_0" + assert map_license(KaggleImporter._license({"licenseName": "ODC-By-1.0"})) == ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION" + ) From d99454a49f2f8e21be7c66627dcc8c4ced570453 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 2 Nov 2026 16:10:10 +0530 Subject: [PATCH 31/55] feat(api): add event search with filters and pagination - /api/search/event/: published events only; filters location, is_individual_event, timeframe (upcoming/past); sorts alphabetical, recent, created, start_date; aggregations for status, location, is_individual_event. - EventDocument (index "event") and event signals: publish adds, rename updates, unpublish/delete removes; search cache refreshed. - Unified search: types=event, title/description normalised. - sync_search_mappings creates a missing index with its full mapping instead of skipping it, so a new index never gets guessed field types. --- DataSpace/settings.py | 1 + .../commands/sync_search_mappings.py | 16 +- api/signals/__init__.py | 1 + api/signals/event_signals.py | 90 ++++++++++ api/urls.py | 2 + api/views/paginated_elastic_view.py | 14 ++ api/views/search_event.py | 166 ++++++++++++++++++ api/views/search_unified.py | 28 ++- search/documents/__init__.py | 1 + search/documents/event_document.py | 112 ++++++++++++ 10 files changed, 425 insertions(+), 6 deletions(-) create mode 100644 api/signals/event_signals.py create mode 100644 api/views/search_event.py create mode 100644 search/documents/event_document.py diff --git a/DataSpace/settings.py b/DataSpace/settings.py index d390827e..2060adb1 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -288,6 +288,7 @@ "search.documents.aimodel_document": "aimodel", "search.documents.publication_document": "publication", "search.documents.collaborative_document": "collaborative", + "search.documents.event_document": "event", "search.documents.publisher_document.OrganizationPublisherDocument": "organization_publisher", "search.documents.publisher_document.UserPublisherDocument": "user_publisher", } diff --git a/api/management/commands/sync_search_mappings.py b/api/management/commands/sync_search_mappings.py index 73552df6..b1d4e725 100644 --- a/api/management/commands/sync_search_mappings.py +++ b/api/management/commands/sync_search_mappings.py @@ -18,9 +18,12 @@ background on the Elasticsearch side (no rebuild, no downtime; until it finishes, searches on the new sub-fields simply match fewer documents). -It skips indexes that do not exist yet (those are created with the full mapping -on first use) and never fails the process: problems are logged and reported, -so a search outage cannot stop the backend from starting. +A registered document whose index does not exist yet (a new search type, such +as events) gets its index created, empty, with the full mapping and analysers. +Without that, the first published item would make Elasticsearch create the +index itself with guessed field types, and aggregations and sorting on it would +fail. The command never fails the process: problems are logged and reported, so +a search outage cannot stop the backend from starting. python manage.py sync_search_mappings # apply python manage.py sync_search_mappings --dry-run # report only @@ -77,7 +80,12 @@ def handle(self, *args: Any, **options: Any) -> None: try: es = document._get_connection() if not es.indices.exists(index=index_name): - self.stdout.write(f"{index_name}: no index yet, skipped") + if dry_run: + self.stdout.write(f"{index_name}: would create the index") + continue + document._index.create(using=es) + self.stdout.write(self.style.SUCCESS(f"{index_name}: created the index")) + logger.info("search_index_created", index=index_name) continue live = es.indices.get_mapping(index=index_name) existing: Dict[str, Any] = {} diff --git a/api/signals/__init__.py b/api/signals/__init__.py index 542aabaf..3d504230 100644 --- a/api/signals/__init__.py +++ b/api/signals/__init__.py @@ -2,6 +2,7 @@ from api.signals import ( aimodel_signals, dataset_signals, + event_signals, publication_signals, usecase_signals, ) diff --git a/api/signals/event_signals.py b/api/signals/event_signals.py new file mode 100644 index 00000000..01a50b41 --- /dev/null +++ b/api/signals/event_signals.py @@ -0,0 +1,90 @@ +""" +event_signals +───────────── +Keep search honest: publishing an event adds it to the index, unpublishing or +deleting it removes it, so a draft is never searchable. Each index write is +followed by a search-cache bump so listings do not serve stale pages. +""" + +from typing import Any + +import structlog +from django.db.models.signals import post_delete, pre_save +from django.dispatch import receiver + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.search_cache import invalidate_search_cache +from search.documents import EventDocument + +logger = structlog.getLogger(__name__) + + +def _should_be_indexed(instance: Event) -> bool: + """An event belongs in search only while it is PUBLISHED.""" + return instance.status == EventStatus.PUBLISHED.value + + +@receiver(pre_save, sender=Event) +def handle_event_visibility(sender: Any, instance: Event, **kwargs: Any) -> None: + """Add / refresh / drop the search document as an event is published or not. + + New rows are handled by the django-elasticsearch-dsl signal processor. ES + errors are logged and swallowed — indexing is best-effort and a rebuild + reconciles it. + """ + if not instance.pk: + return + + try: + original = Event.objects.get(pk=instance.pk) + except Event.DoesNotExist: + return + + was_indexable = _should_be_indexed(original) + is_indexable = _should_be_indexed(instance) + + if was_indexable and is_indexable: + action = "update" + elif was_indexable and not is_indexable: + action = "delete" + elif not was_indexable and is_indexable: + action = "add" + else: + return + + try: + document = EventDocument.get(id=instance.id, ignore=404) + if action == "delete": + if document: + document.delete(refresh=True) + logger.info("event removed from search index", event_id=str(instance.id)) + else: + EventDocument().update(instance, refresh=True) + logger.info("event synced to search index", event_id=str(instance.id), action=action) + # After the index write, never before it: see invalidate_search_cache. + invalidate_search_cache() + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to sync event search document", + event_id=str(instance.id), + action=action, + error=str(exc), + ) + + +@receiver(post_delete, sender=Event) +def remove_event_document(sender: Any, instance: Event, **kwargs: Any) -> None: + """Drop the search document when an event is deleted.""" + try: + document = EventDocument.get(id=instance.id, ignore=404) + if document: + document.delete(refresh=True) + logger.info("deleted event removed from search index", event_id=str(instance.id)) + invalidate_search_cache() + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to delete event search document", + event_id=str(instance.id), + error=str(exc), + ) diff --git a/api/urls.py b/api/urls.py index fbd64b64..f2170425 100644 --- a/api/urls.py +++ b/api/urls.py @@ -19,6 +19,7 @@ search_aimodel, search_collaborative, search_dataset, + search_event, search_publication, search_publisher, search_unified, @@ -64,6 +65,7 @@ search_collaborative.SearchCollaborative.as_view(), name="search_collaborative", ), + path("search/event/", search_event.SearchEvent.as_view(), name="search_event"), path("search/unified/", search_unified.UnifiedSearch.as_view(), name="search_unified"), path("search/publisher/", search_publisher.SearchPublisher.as_view(), name="search_publisher"), path( diff --git a/api/views/paginated_elastic_view.py b/api/views/paginated_elastic_view.py index 9f81f09a..0277a071 100644 --- a/api/views/paginated_elastic_view.py +++ b/api/views/paginated_elastic_view.py @@ -186,6 +186,20 @@ def get(self, request: HttpRequest) -> Response: for agg in running_status_agg: aggregations["running_status"][agg["key"]] = agg["doc_count"] + if "location" in aggregations: + location_agg = aggregations["location"]["buckets"] + aggregations.pop("location") + aggregations["location"] = {} + for agg in location_agg: + aggregations["location"][agg["key"]] = agg["doc_count"] + + if "is_individual_event" in aggregations: + is_individual_event_agg = aggregations["is_individual_event"]["buckets"] + aggregations.pop("is_individual_event") + aggregations["is_individual_event"] = {} + for agg in is_individual_event_agg: + aggregations["is_individual_event"][agg["key_as_string"]] = agg["doc_count"] + if "is_individual_usecase" in aggregations: is_individual_usecase_agg = aggregations["is_individual_usecase"]["buckets"] aggregations.pop("is_individual_usecase") diff --git a/api/views/search_event.py b/api/views/search_event.py new file mode 100644 index 00000000..6fe25e7c --- /dev/null +++ b/api/views/search_event.py @@ -0,0 +1,166 @@ +"""Search view for Event using Elasticsearch. + +Only published events are in the index (the document's ``should_index_object`` +gate), so this public endpoint never leaks drafts. Filters: location, owner +type (``is_individual_event``) and ``timeframe`` (upcoming / past). +""" + +from typing import Any, Dict, List, Optional, Tuple, Union + +import structlog +from elasticsearch_dsl import Q as ESQ +from elasticsearch_dsl import Search +from elasticsearch_dsl.query import Query as ESQuery +from rest_framework import serializers +from rest_framework.permissions import AllowAny + +from api.models.Event import Event +from api.utils.search_query import nested_paths, text_query +from api.utils.telemetry_utils import trace_method +from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView +from search.documents import EventDocument + +logger = structlog.get_logger(__name__) + + +def timeframe_query(timeframe: str) -> Optional[ESQuery]: + """Upcoming = not ended yet; past = already ended (dates compared to today, UTC). + + An event's end is its end date, or its start date when it has none. Events + with no dates at all match neither. + """ + if timeframe == "upcoming": + end, start = {"gte": "now/d"}, {"gte": "now/d"} + elif timeframe == "past": + end, start = {"lt": "now/d"}, {"lt": "now/d"} + else: + return None + return ESQ( + "bool", + should=[ + ESQ("range", end_date=end), + ESQ( + "bool", + must_not=[ESQ("exists", field="end_date")], + filter=[ESQ("range", start_date=start)], + ), + ], + minimum_should_match=1, + ) + + +class EventDocumentSerializer(serializers.ModelSerializer): + """Serializer for the Event search document.""" + + public_id = serializers.CharField() + url_slug = serializers.CharField() + # Elasticsearch hands dates back as datetimes; show them as plain dates. + start_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) + end_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) + is_individual_event = serializers.BooleanField() + + class OrganizationSerializer(serializers.Serializer): + name = serializers.CharField() + logo = serializers.CharField() + + class UserSerializer(serializers.Serializer): + name = serializers.CharField() + bio = serializers.CharField() + profile_picture = serializers.CharField() + + organization = OrganizationSerializer(allow_null=True) + user = UserSerializer(allow_null=True) + + class Meta: + model = Event + fields = [ + "id", + "name", + "public_id", + "url_slug", + "location", + "start_date", + "end_date", + "status", + "is_individual_event", + "created", + "modified", + "organization", + "user", + ] + + +class SearchEvent(PaginatedElasticSearchAPIView): + """View for searching events.""" + + serializer_class = EventDocumentSerializer + document_class = EventDocument + permission_classes = [AllowAny] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.searchable_fields: List[str] + self.aggregations: Dict[str, str] + self.searchable_fields, self.aggregations = self.get_searchable_and_aggregations() + self.logger = structlog.get_logger(__name__) + + @trace_method( + name="get_searchable_and_aggregations", + attributes={"component": "search_event"}, + ) + def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: + """Searchable fields (name/location/owner) and the facet aggregations.""" + searchable_fields: List[str] = ["name", "location", "organization.name", "user.name"] + aggregations: Dict[str, str] = { + "status": "terms", + "location.raw": "terms", + "is_individual_event": "terms", + } + return searchable_fields, aggregations + + @trace_method(name="add_aggregations", attributes={"component": "search_event"}) + def add_aggregations(self, search: Search) -> Search: + """Add the facet aggregations to the search query.""" + for aggregation_field in self.aggregations: + search.aggs.bucket( + aggregation_field.replace(".raw", ""), + self.aggregations[aggregation_field], + field=aggregation_field, + ) + return search + + @trace_method(name="generate_q_expression", attributes={"component": "search_event"}) + def generate_q_expression(self, query: str) -> Optional[Union[ESQuery, List[ESQuery]]]: + """Analysed text query over the searchable fields (api/utils/search_query.py).""" + return text_query(query, self.searchable_fields, nested=nested_paths(self.document_class)) + + @trace_method(name="add_filters", attributes={"component": "search_event"}) + def add_filters(self, filters: Dict[str, str], search: Search) -> Search: + """Apply location / owner-type / timeframe filters.""" + for filter_key in filters: + if filter_key == "location": + filter_values = filters[filter_key].split(",") + search = search.filter("terms", **{"location.raw": filter_values}) + elif filter_key == "is_individual_event": + value = filters[filter_key].lower() == "true" + search = search.filter("term", is_individual_event=value) + elif filter_key == "timeframe": + timeframe = timeframe_query(filters[filter_key]) + if timeframe is not None: + search = search.filter(timeframe) + elif filter_key == "status": + search = search.filter("term", **{filter_key: filters[filter_key]}) + return search + + @trace_method(name="add_sort", attributes={"component": "search_event"}) + def add_sort(self, sort: str, search: Search, order: str) -> Search: + """Apply a sort mode (alphabetical / recent / created / start_date).""" + if sort == "alphabetical": + search = search.sort({"name.raw": {"order": order}}) + elif sort == "recent": + search = search.sort({"modified": {"order": order}}) + elif sort == "created": + search = search.sort({"created": {"order": order}}) + elif sort == "start_date": + search = search.sort({"start_date": {"order": order, "missing": "_last"}}) + return search diff --git a/api/views/search_unified.py b/api/views/search_unified.py index 1323f5ec..4f09c06d 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -85,6 +85,14 @@ class UserSerializer(serializers.Serializer): resource_type = serializers.CharField(required=False) is_individual_publication = serializers.BooleanField(required=False) + # Event specific + public_id = serializers.CharField(required=False) + url_slug = serializers.CharField(required=False) + location = serializers.CharField(required=False, allow_blank=True) + start_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) + end_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) + is_individual_event = serializers.BooleanField(required=False) + # Collaborative specific is_individual_collaborative = serializers.BooleanField(required=False) website = serializers.CharField(required=False) @@ -149,6 +157,12 @@ def _get_index_names(self, types_list: List[str]) -> List[str]: ) index_names.append(collaborative_index) + if "event" in types_list: + event_index = settings.ELASTICSEARCH_INDEX_NAMES.get( + "search.documents.event_document", "event" + ) + index_names.append(event_index) + if "publisher" in types_list: org_publisher_index = settings.ELASTICSEARCH_INDEX_NAMES.get( "search.documents.publisher_document.OrganizationPublisherDocument", @@ -327,6 +341,8 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["type"] = "publication" elif "collaborative" in index_name: result["type"] = "collaborative" + elif "event" in index_name: + result["type"] = "event" elif "publisher" in index_name: result["type"] = "publisher" else: @@ -356,6 +372,12 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["description"] = result.get("summary", "") if "title" not in result: result["title"] = "" + elif result["type"] == "event": + result["title"] = result.get("name", "") + result["description"] = result.get("location") or "" + result.setdefault("tags", []) + result.setdefault("sectors", []) + result.setdefault("geographies", []) elif result["type"] == "publisher": if "name" in result: result["title"] = result.get("name", "") @@ -441,6 +463,8 @@ def perform_unified_search( aggregations["types"]["publication"] = bucket["doc_count"] elif "collaborative" in index_name: aggregations["types"]["collaborative"] = bucket["doc_count"] + elif "event" in index_name: + aggregations["types"]["event"] = bucket["doc_count"] elif "publisher" in index_name: # Combine both organization and user publisher counts if "publisher" not in aggregations["types"]: @@ -464,7 +488,7 @@ def _generate_unified_cache_key(self, request: Any) -> str: "page": request.GET.get("page", "1"), "size": request.GET.get("size", "10"), "types": request.GET.get( - "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,event,publisher" ), "filters": str(sorted(request.GET.dict().items())), "version": str(cache.get(SEARCH_CACHE_VERSION_KEY, 0)), @@ -484,7 +508,7 @@ def get(self, request: Any) -> Response: page: int = int(request.GET.get("page", 1)) size: int = int(request.GET.get("size", 10)) entity_types: str = request.GET.get( - "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,event,publisher" ) # Which entity types to search types_list = [t.strip() for t in entity_types.split(",")] diff --git a/search/documents/__init__.py b/search/documents/__init__.py index 4c6a678a..b679fb3e 100644 --- a/search/documents/__init__.py +++ b/search/documents/__init__.py @@ -1,6 +1,7 @@ from search.documents.aimodel_document import AIModelDocument from search.documents.collaborative_document import CollaborativeDocument from search.documents.dataset_document import DatasetDocument +from search.documents.event_document import EventDocument from search.documents.publication_document import PublicationDocument from search.documents.publisher_document import ( OrganizationPublisherDocument, diff --git a/search/documents/event_document.py b/search/documents/event_document.py new file mode 100644 index 00000000..f0043d0d --- /dev/null +++ b/search/documents/event_document.py @@ -0,0 +1,112 @@ +"""Elasticsearch document for Event. + +Indexes published events only: name, location, dates, link (``public_id`` / +``url_slug``) and owner. The Luma embed is not indexed; the event page fetches +it with ``eventBySlug``. +""" + +from typing import Any, Dict, List, Optional, Union + +from django_elasticsearch_dsl import Document, Index, KeywordField, fields + +from api.models.Event import Event +from api.models.Organization import Organization +from api.utils.enums import EventStatus +from authorization.models import User +from DataSpace import settings +from search.documents.analysers import ngram_analyser, title_subfields + +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) +INDEX.settings(number_of_shards=1, number_of_replicas=0) + + +@INDEX.doc_type +class EventDocument(Document): + """Elasticsearch document for a published event.""" + + name = fields.TextField( + analyzer=ngram_analyser, + fields=title_subfields(), + ) + location = fields.TextField( + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + status = fields.KeywordField() + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") + is_individual_event = fields.BooleanField(attr="is_individual_event") + + organization = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "logo": fields.TextField(analyzer=ngram_analyser), + } + ) + user = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "bio": fields.TextField(analyzer=ngram_analyser), + "profile_picture": fields.TextField(analyzer=ngram_analyser), + } + ) + + def prepare_location(self, instance: Event) -> Optional[str]: + """Blank locations are left out, so they never show up as a filter option.""" + return instance.location or None + + def prepare_organization(self, instance: Event) -> Optional[Dict[str, str]]: + """Prepare the owning organization for indexing.""" + if instance.organization: + org = instance.organization + return {"name": org.name, "logo": org.logo.url if org.logo else ""} + return None + + def prepare_user(self, instance: Event) -> Optional[Dict[str, str]]: + """Prepare the owning user for indexing.""" + if instance.user: + user = instance.user + return { + "name": user.full_name, + "bio": user.bio or "", + "profile_picture": (user.profile_picture.url if user.profile_picture else ""), + } + return None + + def should_index_object(self, obj: Any) -> bool: + """Only PUBLISHED events are indexed — drafts never reach search.""" + return bool(obj.status == EventStatus.PUBLISHED.value) + + def save(self, *args: Any, **kwargs: Any) -> None: + """Index a published event, or drop it from the index otherwise.""" + if self.should_index_object(self.to_dict()): # type: ignore + super().save(*args, **kwargs) + else: + self.delete(ignore=404) + + def get_queryset(self) -> Any: + """Only published events are populated into the index.""" + return super(EventDocument, self).get_queryset().filter(status=EventStatus.PUBLISHED) + + def get_instances_from_related( + self, related_instance: Union[Organization, User] + ) -> Optional[List[Event]]: + """Re-index an owner's events when the owner's name or picture changes.""" + if isinstance(related_instance, (Organization, User)): + return list(related_instance.events.all()) + return None + + class Django: + """Django model configuration.""" + + model = Event + + fields = [ + "id", + "start_date", + "end_date", + "created", + "modified", + ] + + related_models = [Organization, User] From 4a38e322af62121714674ca30d598d5697965788 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 2 Nov 2026 17:22:23 +0530 Subject: [PATCH 32/55] fix(ci): create the event search index on deploy The release step creates each search index from a fixed model list and then populates every index. api.Event was missing from the list, so the populate wrote dev's published events into a missing index and Elasticsearch created it with guessed field types; /api/search/event/ then failed on every request. Creating it from the list gives it the real mapping before anything is indexed. --- scripts/ci-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index 3f3aab65..aa9bc354 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -134,7 +134,7 @@ $RELEASE migrate --noinput # exists" on its own, gets the correctness of both without either failure # mode. Keep this list in sync with DataSpace/settings.py's # ELASTICSEARCH_INDEX_NAMES if a new search document is ever added. -for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Organization authorization.User; do +for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Event api.Organization authorization.User; do $RELEASE search_index --create --models "$model" -f || true done $RELEASE search_index --populate -f From 59fe95097a94ff04e1d09c6fb86dabd0a89c74ca Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 2 Nov 2026 18:25:49 +0530 Subject: [PATCH 33/55] fix(api): rebuild the event search index once if it has guessed types The first event search deploy populated the event index before it was created, so Elasticsearch guessed its field types and /api/search/event/ fails on every request. This one-time migration drops the index when its status field is not a keyword. The deploy runs migrations before it creates and populates the search indexes, so the same deploy recreates it with the real mapping and refills it. No-op when the index is missing or correct; Elasticsearch errors are logged and never fail the migration. --- .../0057_repair_event_search_index.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 api/migrations/0057_repair_event_search_index.py diff --git a/api/migrations/0057_repair_event_search_index.py b/api/migrations/0057_repair_event_search_index.py new file mode 100644 index 00000000..98c333ad --- /dev/null +++ b/api/migrations/0057_repair_event_search_index.py @@ -0,0 +1,47 @@ +"""One-time repair of the ``event`` search index. Touches Elasticsearch only. + +The first deploy with event search populated the index before it was created +(``api.Event`` was missing from the create list in scripts/ci-deploy.sh, fixed +since), so Elasticsearch created it with guessed field types and every event +search failed. If the live index has such a guessed mapping (``status`` not a +keyword), drop it: the deploy's release step runs migrations first, then +creates each search index and populates it, so the next steps rebuild it with +the real mapping and refill it from the database. + +Does nothing when the index is missing or already correct, and never fails the +migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def drop_guessed_event_index(apps, schema_editor): + try: + from elasticsearch_dsl.connections import connections + + index_name = settings.ELASTICSEARCH_INDEX_NAMES["search.documents.event_document"] + es = connections.get_connection().options(request_timeout=10) + if not es.indices.exists(index=index_name): + return + for concrete, body in es.indices.get_mapping(index=index_name).items(): + status = body.get("mappings", {}).get("properties", {}).get("status", {}) + if status.get("type") != "keyword": + es.indices.delete(index=concrete) + logger.info("guessed_event_index_dropped", index=concrete) + except Exception as exc: # never block a deploy on search problems + logger.error("event_index_repair_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0056_update_license_choices"), + ] + + operations = [ + migrations.RunPython(drop_guessed_event_index, migrations.RunPython.noop), + ] From 69424d56cff19251dc6962bf00771c37ca94ee5f Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Tue, 3 Nov 2026 16:00:25 +0530 Subject: [PATCH 34/55] Merge branch 'dev' into s3_migration --- .../{0056_file_field_storages.py => 0058_file_field_storages.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename api/migrations/{0056_file_field_storages.py => 0058_file_field_storages.py} (100%) diff --git a/api/migrations/0056_file_field_storages.py b/api/migrations/0058_file_field_storages.py similarity index 100% rename from api/migrations/0056_file_field_storages.py rename to api/migrations/0058_file_field_storages.py From 5a7f55ae8cec8629e4a2e17a194b5cc7bda7d490 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Tue, 3 Nov 2026 16:00:40 +0530 Subject: [PATCH 35/55] fix migrations --- api/migrations/0058_file_field_storages.py | 2 +- tests/test_migration_graph.py | 26 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/test_migration_graph.py diff --git a/api/migrations/0058_file_field_storages.py b/api/migrations/0058_file_field_storages.py index afbc3380..26ed586d 100644 --- a/api/migrations/0058_file_field_storages.py +++ b/api/migrations/0058_file_field_storages.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0055_event"), + ("api", "0057_repair_event_search_index"), ] operations = [ diff --git a/tests/test_migration_graph.py b/tests/test_migration_graph.py new file mode 100644 index 00000000..114259e0 --- /dev/null +++ b/tests/test_migration_graph.py @@ -0,0 +1,26 @@ +"""Regression test: the migration graph must have a single leaf per app. + +Root cause: ``0056_file_field_storages`` (s3_migration) and ``0056_update_license_choices`` +(dev) both depended on ``0055_event``, and dev then added ``0057`` on top of its own branch. +Merging dev left two leaf nodes, so ``manage.py migrate`` aborted with "Conflicting migrations +detected" and the container never started. The suite did not notice because ``--nomigrations`` +hides the real migration modules. + +Fix: renumber the branch's migration after the latest one on dev and fix its ``dependencies``. +""" + +from django.db.migrations.loader import MigrationLoader +from django.test import override_settings + + +# pytest's --nomigrations disables MIGRATION_MODULES, which would leave an empty graph here and +# make these tests pass whatever is on disk, so restore the real modules. +@override_settings(MIGRATION_MODULES={}) +def test_migration_graph_has_no_conflicts() -> None: + assert MigrationLoader(None).detect_conflicts() == {} + + +@override_settings(MIGRATION_MODULES={}) +def test_migration_graph_is_actually_loaded() -> None: + # Guards the test above against passing on an empty graph. + assert MigrationLoader(None).graph.leaf_nodes("api") From 4d90ade3e07aa1f72be12399e30a5032785d66ae Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 3 Nov 2026 16:19:52 +0530 Subject: [PATCH 36/55] fix(ci): pass STORAGE_* vars to the image sanity check Settings now require STORAGE_BUCKET/REGION/ENV (#249), so manage.py check failed with KeyError and blocked the dev deploy. manage.py check makes no S3 calls, so placeholder values are enough. --- .github/workflows/deploy-backend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index ce870818..b723adeb 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -185,6 +185,9 @@ jobs: -e SECRET_KEY=ci-sanity-check-not-a-real-key \ -e URL_WHITELIST=http://localhost \ -e DB_ENGINE=django.db.backends.sqlite3 \ + -e STORAGE_BUCKET=ci-sanity-check \ + -e STORAGE_REGION=ap-south-1 \ + -e STORAGE_ENV=ci \ --entrypoint python \ "${{ steps.ref.outputs.image_ref }}" \ manage.py check; then From f2f18c357d596c2fe750f38d64d9e2d116a47fb6 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 3 Nov 2026 16:25:55 +0530 Subject: [PATCH 37/55] feat(api): add Luma cover image to event search results The consumer event list needs an image per card. Luma has no image in our data, so the event search document now looks it up while it is built: it takes the evt- id from the embed URL stored on the event, calls https://api2.luma.com/event/get and indexes event.cover_url as cover_url (null when there is no embed, no image or the call fails). The lookup runs on every index write, including search_index --populate on deploy, so no value is carried over from a previous index. Only that fixed Luma URL is called (strict id, no redirects, 3 s timeout, 1 MB cap) and only https images on images.lumacdn.com are accepted. cover_url is returned by /api/search/event/ and by unified search; the embed itself is not. Migration 0059 touches Elasticsearch only: it adds cover_url to an existing event index, because the deploy refills indexes before sync_search_mappings runs and Elasticsearch would otherwise guess the field type. --- api/migrations/0059_event_search_cover_url.py | 40 ++++ api/utils/luma_cover.py | 95 ++++++++++ api/views/search_event.py | 3 + api/views/search_unified.py | 2 + search/documents/event_document.py | 13 +- tests/schema/test_event_cover.py | 175 ++++++++++++++++++ tests/schema/test_event_schema.py | 7 + 7 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 api/migrations/0059_event_search_cover_url.py create mode 100644 api/utils/luma_cover.py create mode 100644 tests/schema/test_event_cover.py diff --git a/api/migrations/0059_event_search_cover_url.py b/api/migrations/0059_event_search_cover_url.py new file mode 100644 index 00000000..fc051a97 --- /dev/null +++ b/api/migrations/0059_event_search_cover_url.py @@ -0,0 +1,40 @@ +"""Add ``cover_url`` to a live ``event`` search index. Touches Elasticsearch only. + +The deploy's release step refills every index right after ``migrate`` and +before ``sync_search_mappings`` runs. Without this step Elasticsearch would +guess the type of the new ``cover_url`` field on the first refilled document. +Does nothing when the index is missing (the release step then creates it with +the full mapping), and never fails the migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def add_cover_url_field(apps, schema_editor): + try: + from elasticsearch_dsl.connections import connections + + index_name = settings.ELASTICSEARCH_INDEX_NAMES["search.documents.event_document"] + es = connections.get_connection().options(request_timeout=10) + if es.indices.exists(index=index_name): + es.indices.put_mapping( + index=index_name, + properties={"cover_url": {"type": "keyword", "index": False}}, + ) + except Exception as exc: # never block a deploy on search problems + logger.error("event_cover_mapping_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0058_file_field_storages"), + ] + + operations = [ + migrations.RunPython(add_cover_url_field, migrations.RunPython.noop), + ] diff --git a/api/utils/luma_cover.py b/api/utils/luma_cover.py new file mode 100644 index 00000000..7a72400d --- /dev/null +++ b/api/utils/luma_cover.py @@ -0,0 +1,95 @@ +""" +luma_cover +────────── +Look up the cover image of a Luma event from its stored embed URL. + +Used while building the event search document, so every index write +(publish, edit, ``search_index --populate`` on deploy) asks Luma afresh; no +value is carried over from a previous index. + +- The event id is taken from our own stored embed ``src`` + (``…/embed/event/evt-…``) and must match a strict pattern. +- The request always goes to ``https://api2.luma.com/event/get`` with that id; + redirects are not followed and at most ``_MAX_BYTES`` are read. +- Only an ``https`` image on ``images.lumacdn.com`` is accepted. + +Any failure (no embed, timeout, error status, unexpected body, no image) +returns None and is logged; indexing never fails because of Luma. +""" + +import json +import re +from typing import Any, Optional +from urllib.parse import urlparse + +import requests +import structlog + +logger = structlog.get_logger(__name__) + +_EVENT_ID = re.compile(r"^/embed/event/(evt-[A-Za-z0-9]{1,64})(?:/|$)") +_API_URL = "https://api2.luma.com/event/get" +_IMAGE_HOST = "images.lumacdn.com" +_MAX_BYTES = 1_000_000 +_TIMEOUT = 3 + + +def luma_event_id(src: Optional[str]) -> Optional[str]: + """Return the ``evt-…`` id from a Luma event embed URL, or None.""" + if not isinstance(src, str) or not src: + return None + match = _EVENT_ID.match(urlparse(src).path) + return match.group(1) if match else None + + +def cover_from_response(data: Any, event_id: str) -> Optional[str]: + """Return ``event.cover_url`` from an ``event/get`` response body, or None.""" + event = data.get("event") if isinstance(data, dict) else None + if not isinstance(event, dict) or event.get("api_id") != event_id: + return None + return _safe_image_url(event.get("cover_url")) + + +def fetch_luma_cover(embed_src: Optional[str]) -> Optional[str]: + """Return the cover image URL of the Luma event in ``embed_src``, or None.""" + event_id = luma_event_id(embed_src) + if not event_id: + return None + try: + with requests.get( + _API_URL, + params={"event_api_id": event_id}, + timeout=_TIMEOUT, + allow_redirects=False, + stream=True, + headers={"User-Agent": "DataSpace/1.0", "Accept": "application/json"}, + ) as response: + if response.status_code != 200: + logger.warning( + "luma_cover_fetch_failed", event_id=event_id, status=response.status_code + ) + return None + body = b"" + for chunk in response.iter_content(chunk_size=65536): + body += chunk + if len(body) > _MAX_BYTES: + logger.warning("luma_cover_response_too_large", event_id=event_id) + return None + cover = cover_from_response(json.loads(body), event_id) + except (requests.RequestException, ValueError) as exc: + logger.warning("luma_cover_fetch_failed", event_id=event_id, error=type(exc).__name__) + return None + if not cover: + logger.info("luma_cover_not_found", event_id=event_id) + return cover + + +def _safe_image_url(value: Any) -> Optional[str]: + if not isinstance(value, str) or not value or len(value) > 500: + return None + parsed = urlparse(value) + if parsed.scheme != "https" or parsed.hostname != _IMAGE_HOST: + return None + if parsed.username or parsed.password or parsed.port: + return None + return value diff --git a/api/views/search_event.py b/api/views/search_event.py index 6fe25e7c..40c54757 100644 --- a/api/views/search_event.py +++ b/api/views/search_event.py @@ -58,6 +58,8 @@ class EventDocumentSerializer(serializers.ModelSerializer): start_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) end_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) is_individual_event = serializers.BooleanField() + # Luma cover image (https URL on images.lumacdn.com), null when unknown. + cover_url = serializers.CharField(default=None, allow_null=True) class OrganizationSerializer(serializers.Serializer): name = serializers.CharField() @@ -83,6 +85,7 @@ class Meta: "end_date", "status", "is_individual_event", + "cover_url", "created", "modified", "organization", diff --git a/api/views/search_unified.py b/api/views/search_unified.py index 4f09c06d..633cda3e 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -92,6 +92,7 @@ class UserSerializer(serializers.Serializer): start_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) end_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) is_individual_event = serializers.BooleanField(required=False) + cover_url = serializers.CharField(required=False, allow_null=True) # Collaborative specific is_individual_collaborative = serializers.BooleanField(required=False) @@ -375,6 +376,7 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: elif result["type"] == "event": result["title"] = result.get("name", "") result["description"] = result.get("location") or "" + result.setdefault("cover_url", None) result.setdefault("tags", []) result.setdefault("sectors", []) result.setdefault("geographies", []) diff --git a/search/documents/event_document.py b/search/documents/event_document.py index f0043d0d..565105dd 100644 --- a/search/documents/event_document.py +++ b/search/documents/event_document.py @@ -1,8 +1,10 @@ """Elasticsearch document for Event. Indexes published events only: name, location, dates, link (``public_id`` / -``url_slug``) and owner. The Luma embed is not indexed; the event page fetches -it with ``eventBySlug``. +``url_slug``), owner and ``cover_url``. The cover is looked up on Luma every +time the document is built (``prepare_cover_url``), including the deploy's +``search_index --populate``; it is display-only (not searchable). The Luma +embed itself is not indexed; the event page fetches it with ``eventBySlug``. """ from typing import Any, Dict, List, Optional, Union @@ -12,6 +14,7 @@ from api.models.Event import Event from api.models.Organization import Organization from api.utils.enums import EventStatus +from api.utils.luma_cover import fetch_luma_cover from authorization.models import User from DataSpace import settings from search.documents.analysers import ngram_analyser, title_subfields @@ -36,6 +39,7 @@ class EventDocument(Document): public_id = fields.KeywordField() url_slug = fields.KeywordField(attr="url_slug") is_individual_event = fields.BooleanField(attr="is_individual_event") + cover_url = fields.KeywordField(index=False) organization = fields.NestedField( properties={ @@ -55,6 +59,11 @@ def prepare_location(self, instance: Event) -> Optional[str]: """Blank locations are left out, so they never show up as a filter option.""" return instance.location or None + def prepare_cover_url(self, instance: Event) -> Optional[str]: + """Cover image from Luma for the stored embed; None when Luma has none or fails.""" + embed = instance.luma_embed if isinstance(instance.luma_embed, dict) else {} + return fetch_luma_cover(embed.get("src")) + def prepare_organization(self, instance: Event) -> Optional[Dict[str, str]]: """Prepare the owning organization for indexing.""" if instance.organization: diff --git a/tests/schema/test_event_cover.py b/tests/schema/test_event_cover.py new file mode 100644 index 00000000..138e4d0f --- /dev/null +++ b/tests/schema/test_event_cover.py @@ -0,0 +1,175 @@ +"""Luma cover image on the event search document.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.luma_cover import cover_from_response, fetch_luma_cover, luma_event_id +from search.documents import EventDocument + +EMBED = "https://lu.ma/embed/event/evt-abc123/simple" +COVER = "https://images.lumacdn.com/gallery-images/rc/5f979993.png" +NEW_COVER = "https://images.lumacdn.com/gallery-images/7t/2758125c.png" + + +def body(event: object) -> dict: + return {"api_id": "x", "event": event} + + +def response(status: int = 200, payload: object = None, raw: bytes = b"") -> MagicMock: + resp = MagicMock() + resp.status_code = status + resp.iter_content.return_value = [raw or json.dumps(payload).encode()] + resp.__enter__.return_value = resp + return resp + + +# ── embed id ───────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "src, expected", + [ + (EMBED, "evt-abc123"), + ("https://luma.com/embed/event/evt-abc123", "evt-abc123"), + ("https://luma.com/embed/calendar/cal-x/events", None), + ("https://luma.com/embed/event/../../admin/simple", None), + ("", None), + (None, None), + (123, None), + ], +) +def test_event_id_comes_only_from_an_event_embed(src, expected): + assert luma_event_id(src) == expected + + +# ── response parsing ───────────────────────────────────────────────────── + + +def test_cover_is_read_from_the_event(): + assert ( + cover_from_response(body({"api_id": "evt-abc123", "cover_url": COVER}), "evt-abc123") + == COVER + ) + + +def test_cover_of_another_event_is_ignored(): + assert ( + cover_from_response(body({"api_id": "evt-zzz", "cover_url": COVER}), "evt-abc123") is None + ) + + +@pytest.mark.parametrize( + "url", + [ + "http://images.lumacdn.com/x.png", + "https://evil.example.com/x.png", + "https://images.lumacdn.com.evil.example/x.png", + "https://user@images.lumacdn.com/x.png", + "", + None, + 123, + ], +) +def test_only_https_luma_cdn_images_are_accepted(url): + assert ( + cover_from_response(body({"api_id": "evt-abc123", "cover_url": url}), "evt-abc123") is None + ) + + +@pytest.mark.parametrize("data", [{}, [], "x", None, body(None), body("x")]) +def test_unexpected_body_gives_no_cover(data): + assert cover_from_response(data, "evt-abc123") is None + + +# ── fetching ───────────────────────────────────────────────────────────── + + +@patch("api.utils.luma_cover.requests.get") +def test_fetch_calls_event_get_with_the_id_without_redirects(get): + get.return_value = response(payload=body({"api_id": "evt-abc123", "cover_url": COVER})) + assert fetch_luma_cover(EMBED) == COVER + assert get.call_args.args[0] == "https://api2.luma.com/event/get" + assert get.call_args.kwargs["params"] == {"event_api_id": "evt-abc123"} + assert get.call_args.kwargs["allow_redirects"] is False + assert get.call_args.kwargs["timeout"] <= 5 + + +@pytest.mark.parametrize( + "resp", + [ + requests.Timeout(), + requests.ConnectionError(), + response(status=404, payload={"message": "not found"}), + response(status=302, payload={}), + response(status=500, payload={}), + response(raw=b"not json"), + ], +) +@patch("api.utils.luma_cover.requests.get") +def test_fetch_failures_give_null(get, resp): + if isinstance(resp, Exception): + get.side_effect = resp + else: + get.return_value = resp + assert fetch_luma_cover(EMBED) is None + + +@patch("api.utils.luma_cover._MAX_BYTES", 10) +@patch("api.utils.luma_cover.requests.get") +def test_oversized_response_is_not_read(get): + get.return_value = response(payload=body({"api_id": "evt-abc123", "cover_url": COVER})) + assert fetch_luma_cover(EMBED) is None + + +@patch("api.utils.luma_cover.requests.get") +def test_no_request_without_an_event_embed(get): + assert fetch_luma_cover(None) is None + assert fetch_luma_cover("https://luma.com/embed/calendar/cal-x/events") is None + get.assert_not_called() + + +# ── search document ────────────────────────────────────────────────────── + + +def event(**kwargs) -> Event: + defaults = dict(name="Meetup", status=EventStatus.PUBLISHED, luma_embed={"src": EMBED}) + defaults.update(kwargs) + return Event(**defaults) + + +@patch("search.documents.event_document.fetch_luma_cover", return_value=COVER) +def test_document_asks_luma_for_the_stored_embed(fetch): + assert EventDocument().prepare(event())["cover_url"] == COVER + fetch.assert_called_once_with(EMBED) + + +@patch("search.documents.event_document.fetch_luma_cover") +def test_document_asks_luma_on_every_build(fetch): + """No value is carried over: each build (publish, edit, deploy populate) asks again.""" + doc, instance = EventDocument(), event() + fetch.return_value = COVER + assert doc.prepare(instance)["cover_url"] == COVER + fetch.return_value = NEW_COVER + assert doc.prepare(instance)["cover_url"] == NEW_COVER + fetch.return_value = None + assert doc.prepare(instance)["cover_url"] is None + assert fetch.call_count == 3 + + +@pytest.mark.parametrize("embed", [{}, None, "not a dict", {"src": None}]) +def test_document_without_embed_has_null_cover(embed): + with patch("api.utils.luma_cover.requests.get") as get: + assert EventDocument().prepare(event(luma_embed=embed))["cover_url"] is None + get.assert_not_called() + + +@patch("search.documents.event_document.fetch_luma_cover", return_value=COVER) +def test_document_does_not_carry_the_embed(fetch): + prepared = EventDocument().prepare(event()) + assert "luma_embed" not in prepared + assert EMBED not in json.dumps(prepared, default=str) diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py index 83edbc57..1b718d71 100644 --- a/tests/schema/test_event_schema.py +++ b/tests/schema/test_event_schema.py @@ -27,6 +27,13 @@ def _no_activity_recording(): yield +@pytest.fixture(autouse=True) +def _no_luma_requests(): + """Publishing builds the search document, which asks Luma for the cover.""" + with patch("search.documents.event_document.fetch_luma_cover", return_value=None): + yield + + @pytest.fixture def roles(db): admin = Role.objects.create( From 30a851d43a0d5cd4e63a7ba0701854431093a5dc Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Mon, 2 Nov 2026 23:37:48 +0530 Subject: [PATCH 38/55] feat(api): permanent public links for all modules --- .../commands/sync_search_mappings.py | 49 ++++- api/migrations/0058_public_ids.py | 71 +++++++ api/migrations/0059_fill_public_ids.py | 20 ++ api/migrations/0060_public_ids_unique.py | 18 ++ api/models/AIModel.py | 9 + api/models/Collaborative.py | 26 +-- api/models/Dataset.py | 21 +- api/models/Publication.py | 19 +- api/models/UseCase.py | 20 +- api/schema/aimodel_schema.py | 18 +- api/schema/collaborative_schema.py | 19 +- api/schema/dataset_schema.py | 45 +++-- api/schema/publication_schema.py | 19 +- api/schema/usecase_schema.py | 18 ++ api/services/metadata_export/adapter.py | 2 +- api/services/platform_import_service.py | 15 -- api/types/type_aimodel.py | 5 + api/types/type_collaborative.py | 6 +- api/types/type_dataset.py | 6 + api/types/type_publication.py | 5 + api/types/type_usecase.py | 6 +- api/utils/django_utils.py | 38 +--- api/views/search_aimodel.py | 5 + api/views/search_collaborative.py | 4 + api/views/search_dataset.py | 4 + api/views/search_publication.py | 5 + api/views/search_unified.py | 2 + api/views/search_usecase.py | 5 + authorization/permissions.py | 17 +- search/documents/aimodel_document.py | 2 + search/documents/collaborative_document.py | 2 + search/documents/dataset_document.py | 2 + search/documents/publication_document.py | 2 + search/documents/usecase_document.py | 2 + tests/schema/test_dataset_public_links.py | 98 ++++++++++ tests/schema/test_module_public_links.py | 179 ++++++++++++++++++ tests/test_public_links.py | 157 +++++++++++++++ tests/test_publication_models.py | 17 +- tests/test_slug_collision.py | 39 ++-- 39 files changed, 839 insertions(+), 158 deletions(-) create mode 100644 api/migrations/0058_public_ids.py create mode 100644 api/migrations/0059_fill_public_ids.py create mode 100644 api/migrations/0060_public_ids_unique.py create mode 100644 tests/schema/test_dataset_public_links.py create mode 100644 tests/schema/test_module_public_links.py create mode 100644 tests/test_public_links.py diff --git a/api/management/commands/sync_search_mappings.py b/api/management/commands/sync_search_mappings.py index b1d4e725..b2351c3c 100644 --- a/api/management/commands/sync_search_mappings.py +++ b/api/management/commands/sync_search_mappings.py @@ -18,6 +18,11 @@ background on the Elasticsearch side (no rebuild, no downtime; until it finishes, searches on the new sub-fields simply match fewer documents). +When top-level fields were added, their values come from the database, which +Elasticsearch cannot compute itself. Existing documents are then updated in +place with just those fields (a partial update per record, not a reindex), so +for example search results for old datasets get ``url_slug`` right away. + A registered document whose index does not exist yet (a new search type, such as events) gets its index created, empty, with the full mapping and analysers. Without that, the first published item would make Elasticsearch create the @@ -33,11 +38,12 @@ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, Iterator, List import structlog from django.core.management.base import BaseCommand from django_elasticsearch_dsl.registries import registry +from elasticsearch.helpers import bulk logger = structlog.get_logger("dataspace.search_mappings") @@ -67,6 +73,39 @@ def missing_subfields(existing: Dict[str, Any], desired: Dict[str, Any]) -> Dict return updates +def refill_fields(document: Any, es: Any, index_name: str, names: List[str]) -> int: + """Write the current values of ``names`` into every existing document. + + Only the given fields are sent (a partial update), computed the same way + the document computes them when a record is saved. Records that are not in + the index (for example ones the document never indexes) are skipped. + Returns the number of documents updated. + """ + doc = document() + preparers = [(name, fn) for name, _field, fn in doc._prepared_fields if name in names] + if not preparers: + return 0 + + def actions() -> Iterator[Dict[str, Any]]: + for instance in doc.get_queryset().iterator(chunk_size=500): + try: + values = {name: fn(instance) for name, fn in preparers} + except Exception as exc: # one bad record must not stop the rest + logger.warning( + "search_refill_skipped", index=index_name, pk=str(instance.pk), error=str(exc) + ) + continue + yield { + "_op_type": "update", + "_index": index_name, + "_id": instance.pk, + "doc": values, + } + + updated, _failed = bulk(es, actions(), raise_on_error=False, stats_only=True) + return int(updated) + + class Command(BaseCommand): help = "Add fields and sub-fields that search documents define but their live indexes lack." @@ -105,8 +144,9 @@ def handle(self, *args: Any, **options: Any) -> None: added += [f"{name}.{sub}" for sub in sorted(spec["fields"]) if sub not in have] names = ", ".join(added) if dry_run: - refill = " and refill existing documents" if subfield_updates else "" - self.stdout.write(f"{index_name}: would add {names}{refill}") + self.stdout.write( + f"{index_name}: would add {names} and refill existing documents" + ) continue es.indices.put_mapping( @@ -120,6 +160,9 @@ def handle(self, *args: Any, **options: Any) -> None: index=index_name, conflicts="proceed", wait_for_completion=False ) message += f"; refilling existing documents (task {task.get('task')})" + if fields_to_add: + count = refill_fields(document, es, index_name, sorted(fields_to_add)) + message += f"; filled new fields in {count} existing documents" self.stdout.write(self.style.SUCCESS(message)) logger.info("search_mapping_fields_added", index=index_name, fields=added) except Exception as exc: # never block start-up on search problems diff --git a/api/migrations/0058_public_ids.py b/api/migrations/0058_public_ids.py new file mode 100644 index 00000000..7b5c26e1 --- /dev/null +++ b/api/migrations/0058_public_ids.py @@ -0,0 +1,71 @@ +import django.core.validators +from django.db import migrations, models + +MODELS = ["aimodel", "collaborative", "dataset", "publication", "usecase"] + + +class Migration(migrations.Migration): + """Add public_id (nullable for now) to every public module. + + Also drops the unique rule on slugs and on use case / collaborative titles: + links use public_id, so titles may repeat. Filled for existing rows in + 0057, made unique and required in 0058. + """ + + dependencies = [ + ("api", "0057_repair_event_search_index"), + ] + + operations = [ + *[ + migrations.AddField( + model_name=model, + name="public_id", + field=models.CharField(editable=False, max_length=12, null=True), + ) + for model in MODELS + ], + migrations.AlterField( + model_name="collaborative", + name="slug", + field=models.SlugField( + blank=True, + max_length=75, + null=True, + validators=[ + django.core.validators.RegexValidator( + message=( + "Slug must be lowercase and contain only alphanumeric " + "characters and hyphens." + ), + regex="^[a-z0-9]+(?:-[a-z0-9]+)*$", + ) + ], + ), + ), + migrations.AlterField( + model_name="collaborative", + name="title", + field=models.CharField(blank=True, max_length=200, null=True), + ), + migrations.AlterField( + model_name="dataset", + name="slug", + field=models.SlugField(max_length=255), + ), + migrations.AlterField( + model_name="publication", + name="slug", + field=models.SlugField(max_length=255), + ), + migrations.AlterField( + model_name="usecase", + name="slug", + field=models.SlugField(blank=True, max_length=75, null=True), + ), + migrations.AlterField( + model_name="usecase", + name="title", + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/api/migrations/0059_fill_public_ids.py b/api/migrations/0059_fill_public_ids.py new file mode 100644 index 00000000..db54577a --- /dev/null +++ b/api/migrations/0059_fill_public_ids.py @@ -0,0 +1,20 @@ +from django.db import migrations + +from api.utils.public_links import fill_missing_public_ids + +MODELS = ["AIModel", "Collaborative", "Dataset", "Publication", "UseCase"] + + +def fill(apps, schema_editor): + for name in MODELS: + fill_missing_public_ids(apps.get_model("api", name)) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0058_public_ids"), + ] + + operations = [ + migrations.RunPython(fill, migrations.RunPython.noop), + ] diff --git a/api/migrations/0060_public_ids_unique.py b/api/migrations/0060_public_ids_unique.py new file mode 100644 index 00000000..f17c6350 --- /dev/null +++ b/api/migrations/0060_public_ids_unique.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + +MODELS = ["aimodel", "collaborative", "dataset", "publication", "usecase"] + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0059_fill_public_ids"), + ] + + operations = [ + migrations.AlterField( + model_name=model, + name="public_id", + field=models.CharField(editable=False, max_length=12, unique=True), + ) + for model in MODELS + ] diff --git a/api/models/AIModel.py b/api/models/AIModel.py index 9646e627..25239028 100644 --- a/api/models/AIModel.py +++ b/api/models/AIModel.py @@ -14,6 +14,7 @@ HFModelClass, PromptDomain, ) +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id User = get_user_model() @@ -27,6 +28,7 @@ class AIModel(models.Model): # Basic Information name = models.CharField(max_length=255, help_text="Model name or identifier") display_name = models.CharField(max_length=255, help_text="Human-readable name") + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) version = models.CharField(max_length=50, blank=True, help_text="Model version") description = models.TextField(help_text="Description of the model's capabilities") @@ -157,6 +159,13 @@ class Meta: def __str__(self): return f"{self.display_name} ({self.provider})" + def save(self, *args, **kwargs): + save_with_public_id(self, lambda: super(AIModel, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + return build_url_slug(self.display_name or self.name, self.public_id) + @property def tags_indexing(self) -> list[str]: """Tags for indexing. diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index f39f04a2..345a4b15 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -10,9 +10,9 @@ from api.models.Organization import Organization from authorization.models import User -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id from api.utils.storage import public_storage slug_validator = RegexValidator( @@ -23,7 +23,7 @@ class Collaborative(models.Model): id = models.AutoField(primary_key=True) - title = models.CharField(max_length=200, unique=True, blank=True, null=True) + title = models.CharField(max_length=200, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( upload_to=_use_case_directory_path, @@ -43,9 +43,9 @@ class Collaborative(models.Model): modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - slug = models.SlugField( - max_length=75, null=True, blank=True, unique=True, validators=[slug_validator] - ) + # Kept for old links and older readers; no longer unique. Links use public_id. + slug = models.SlugField(max_length=75, null=True, blank=True, validators=[slug_validator]) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( "api.Organization", on_delete=models.CASCADE, null=True, blank=True @@ -78,18 +78,18 @@ class Collaborative(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if self.title and not self.slug: - self.slug = slugify(cast(str, self.title)) - base_title = self.title - - def disambiguate(attempt: int) -> None: - self.title = f"{base_title} ({attempt})" - self.slug = slugify(cast(str, self.title)) + self.slug = slugify(cast(str, self.title))[:75].strip("-") def do_save() -> None: - self.full_clean() + # public_id clashes are caught by the database and retried, not here. + self.full_clean(exclude=["public_id"]) super(Collaborative, self).save(*args, **kwargs) - retry_on_slug_collision(do_save, disambiguate) + save_with_public_id(self, do_save) + + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_collaborative(self): diff --git a/api/models/Dataset.py b/api/models/Dataset.py index 296f509c..f3d3386d 100644 --- a/api/models/Dataset.py +++ b/api/models/Dataset.py @@ -5,13 +5,13 @@ from django.db.models import Sum from django.utils.text import slugify -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import ( DatasetAccessType, DatasetLicense, DatasetStatus, DatasetType, ) +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id if TYPE_CHECKING: from api.models.DataSpace import DataSpace @@ -36,7 +36,10 @@ class Dataset(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) title = models.CharField(max_length=300, unique=False, blank=True) description = models.CharField(max_length=1000, unique=False, blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Kept for older readers (SDK, search); no longer unique. Links use + # public_id, see api/utils/public_links.py. + slug = models.SlugField(max_length=255) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) organization = models.ForeignKey( "api.Organization", on_delete=models.SET_NULL, @@ -84,15 +87,13 @@ class Dataset(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - self.slug = slugify(self.title) - base_slug = self.slug + self.slug = slugify(self.title)[:255] + save_with_public_id(self, lambda: super(Dataset, self).save(*args, **kwargs)) - def disambiguate(attempt: int) -> None: - self.slug = f"{base_slug}-{attempt}" - - retry_on_slug_collision( - lambda: super(Dataset, self).save(*args, **kwargs), disambiguate - ) + @property + def url_slug(self) -> str: + """Link part for the dataset page, e.g. ``city-air-quality-index-k3x9m2qa7b1z``.""" + return build_url_slug(self.title, self.public_id) @property def tags_indexing(self) -> list[str]: diff --git a/api/models/Publication.py b/api/models/Publication.py index aea63f46..f1aa1a94 100644 --- a/api/models/Publication.py +++ b/api/models/Publication.py @@ -7,6 +7,7 @@ from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus from api.utils.file_paths import _publication_block_directory_path +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id from api.utils.storage import public_storage if TYPE_CHECKING: @@ -28,7 +29,9 @@ class Publication(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) title = models.CharField(max_length=300, unique=False, blank=True) description = models.TextField(blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Kept for older readers (SDK); no longer unique. Links use public_id (see url_slug). + slug = models.SlugField(max_length=255) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) organization = models.ForeignKey( "api.Organization", @@ -77,14 +80,12 @@ class Publication(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - base_slug = slugify(self.title) or "untitled" - slug = base_slug - counter = 1 - while Publication.objects.filter(slug=slug).exclude(pk=self.pk).exists(): - slug = f"{base_slug}-{counter}" - counter += 1 - self.slug = slug - super().save(*args, **kwargs) + self.slug = slugify(self.title)[:255] or "untitled" + save_with_public_id(self, lambda: super(Publication, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_publication(self) -> bool: diff --git a/api/models/UseCase.py b/api/models/UseCase.py index 0ea9b67d..517531a9 100644 --- a/api/models/UseCase.py +++ b/api/models/UseCase.py @@ -9,19 +9,19 @@ from api.models.Organization import Organization from authorization.models import User -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import ( OrganizationRelationshipType, UseCaseRunningStatus, UseCaseStatus, ) from api.utils.file_paths import _use_case_directory_path +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id from api.utils.storage import public_storage class UseCase(models.Model): id = models.AutoField(primary_key=True) - title = models.CharField(max_length=200, unique=True, blank=True, null=True) + title = models.CharField(max_length=200, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( upload_to=_use_case_directory_path, @@ -34,7 +34,9 @@ class UseCase(models.Model): modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - slug = models.SlugField(max_length=75, null=True, blank=True, unique=True) + # Kept for older readers; no longer unique. Links use public_id (see url_slug). + slug = models.SlugField(max_length=75, null=True, blank=True) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( "api.Organization", on_delete=models.CASCADE, null=True, blank=True @@ -69,14 +71,12 @@ class UseCase(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if self.title and not self.slug: - self.slug = slugify(cast(str, self.title)) - base_title = self.title + self.slug = slugify(cast(str, self.title))[:75].strip("-") + save_with_public_id(self, lambda: super(UseCase, self).save(*args, **kwargs)) - def disambiguate(attempt: int) -> None: - self.title = f"{base_title} ({attempt})" - self.slug = slugify(cast(str, self.title)) - - retry_on_slug_collision(lambda: super(UseCase, self).save(*args, **kwargs), disambiguate) + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_usecase(self): diff --git a/api/schema/aimodel_schema.py b/api/schema/aimodel_schema.py index abd76ae3..7c4655be 100644 --- a/api/schema/aimodel_schema.py +++ b/api/schema/aimodel_schema.py @@ -36,6 +36,7 @@ TypeVersionProvider, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, @@ -427,11 +428,22 @@ def ai_models_table( @strawberry.field @trace_resolver(name="get_ai_model", attributes={"component": "aimodel"}) - def get_ai_model(self, info: Info, model_id: int) -> Optional[TypeAIModel]: - """Get an AI model by ID.""" + def get_ai_model( + self, + info: Info, + model_id: Optional[int] = None, + public_id: Optional[str] = None, + ) -> Optional[TypeAIModel]: + """Get an AI model by ID, or by its link (``urlSlug``) or ``publicId``.""" user = info.context.user try: - model = AIModel.objects.get(id=model_id) + if model_id is not None: + model = AIModel.objects.get(id=model_id) + else: + parsed = parse_public_id(public_id) + if parsed is None: + return None + model = AIModel.objects.get(public_id=parsed) # Check permissions if model.is_public and model.is_active: diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 4a7bed46..58a800d9 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -50,6 +50,7 @@ from api.types.type_usecase import TypeUseCase from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, @@ -293,11 +294,21 @@ def contributors_by_collaborative(self, info: Info, collaborative_id: str) -> li attributes={"component": "collaborative"}, ) def collaborative_by_slug(self, info: Info, slug: str) -> TypeCollaborative: - try: - collaborative = Collaborative.objects.get(slug=slug) - return TypeCollaborative.from_django(collaborative) - except Collaborative.DoesNotExist: + """Collaborative for its page, found by its link (``urlSlug``). + + Only the publicId at the end of the link is used, so a link with old + title words still works. Links shared before public ids existed carry + the old slug, so a value without a matching id falls back to the slug. + """ + public_id = parse_public_id(slug) + collaborative = ( + Collaborative.objects.filter(public_id=public_id).first() if public_id else None + ) + if collaborative is None: + collaborative = Collaborative.objects.filter(slug=slug).order_by("created").first() + if collaborative is None: raise ValueError(f"Collaborative with slug {slug} does not exist.") + return TypeCollaborative.from_django(collaborative) @trace_resolver(name="update_collaborative_tags", attributes={"component": "collaborative"}) diff --git a/api/schema/dataset_schema.py b/api/schema/dataset_schema.py index 2aab35a2..68d7c73f 100644 --- a/api/schema/dataset_schema.py +++ b/api/schema/dataset_schema.py @@ -55,6 +55,7 @@ UseCaseStatus, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, @@ -77,6 +78,14 @@ PromptPurposeENUM = strawberry.enum(PromptPurpose) # type: ignore +def _find_dataset(dataset_id: Optional[uuid.UUID], public_id: Optional[str]) -> Optional[Dataset]: + """The dataset for a UUID or a public link part, or None if there is none.""" + if dataset_id: + return Dataset.objects.filter(id=dataset_id).first() + parsed = parse_public_id(public_id) + return Dataset.objects.filter(public_id=parsed).first() if parsed else None + + # Create permission classes dynamically with different operations class ViewDatasetPermission(DatasetPermissionGraphQL): def __init__(self) -> None: @@ -114,19 +123,19 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: # For queries/mutations that don't have a source yet (e.g., getting a dataset by ID) if source is None: dataset_id = kwargs.get("dataset_id") - if dataset_id: - try: - dataset = Dataset.objects.get(id=dataset_id) - # Allow access to published datasets for everyone - if dataset.status == DatasetStatus.PUBLISHED.value: - return True - except Dataset.DoesNotExist: - pass # Let the resolver handle the non-existent dataset + public_id = kwargs.get("public_id") + if dataset_id or public_id: + dataset = _find_dataset(dataset_id, public_id) + # Allow access to published datasets for everyone + if dataset is not None and dataset.status == DatasetStatus.PUBLISHED.value: + return True # For non-published datasets, require authentication user = request.user if not user.is_authenticated: return False + if dataset is None: + return True # the resolver returns null for a missing dataset if user.is_superuser: return True dataset_perm = DatasetPermission.objects.filter(user=user, dataset=dataset).first() @@ -551,13 +560,19 @@ def datasets_table( permission_classes=[AllowPublishedDatasets], # type: ignore[list-item] ) @trace_resolver(name="get_dataset", attributes={"component": "dataset"}) - def get_dataset(self, info: Info, dataset_id: uuid.UUID) -> Optional[TypeDataset]: - """Get a dataset by ID.""" - try: - dataset = Dataset.objects.get(id=dataset_id) - return TypeDataset.from_django(dataset) - except Dataset.DoesNotExist: - return None + def get_dataset( + self, + info: Info, + dataset_id: Optional[uuid.UUID] = None, + public_id: Optional[str] = None, + ) -> Optional[TypeDataset]: + """Get a dataset by ID, or by its public link. + + ``public_id`` takes the id itself or the whole link part + (``city-air-quality-index-k3x9m2qa7b1z``); only the id at the end is used. + """ + dataset = _find_dataset(dataset_id, public_id) + return TypeDataset.from_django(dataset) if dataset is not None else None @strawberry.field( permission_classes=[ChartDataPermission], # type: ignore[list-item] diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py index 53eecd97..4631ba80 100644 --- a/api/schema/publication_schema.py +++ b/api/schema/publication_schema.py @@ -62,6 +62,7 @@ CreatePublicationPermission, DeletePublicationPermission, PublishPublicationPermission, + find_publication, ) @@ -157,12 +158,18 @@ def resource_types(self, info: Info) -> List[TypeResourceType]: permission_classes=[AllowPublishedPublications], # type: ignore[list-item] ) @trace_resolver(name="get_publication", attributes={"component": "publication"}) - def get_publication(self, info: Info, publication_id: uuid.UUID) -> Optional[TypePublication]: - """Get a single resource by id (drafts gated to owner/org by the permission).""" - try: - return TypePublication.from_django(Publication.objects.get(id=publication_id)) - except Publication.DoesNotExist: - return None + def get_publication( + self, + info: Info, + publication_id: Optional[uuid.UUID] = None, + public_id: Optional[str] = None, + ) -> Optional[TypePublication]: + """Get a single resource by id, or by its link (``urlSlug``) or ``publicId``. + + Drafts are gated to owner/org by the permission. + """ + publication = find_publication(publication_id, public_id) + return TypePublication.from_django(publication) if publication else None @strawberry.field @trace_resolver(name="get_publications", attributes={"component": "publication"}) diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index 6b133d6a..6cf3a22e 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -53,6 +53,7 @@ UseCaseStatus, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, @@ -272,6 +273,23 @@ def published_use_cases( return TypeUseCase.from_django_list(results) + @strawberry_django.field + @trace_resolver(name="get_use_case_by_slug", attributes={"component": "usecase"}) + def use_case_by_slug(self, info: Info, slug: str) -> Optional[TypeUseCase]: + """Published use case for a public page, found by its link (``urlSlug``). + + Only the publicId at the end of the link is used, so a link with old + title words still works. Drafts, unknown ids and links without an id + return null. + """ + public_id = parse_public_id(slug) + if public_id is None: + return None + use_case = UseCase.objects.filter( + public_id=public_id, status=UseCaseStatus.PUBLISHED + ).first() + return TypeUseCase.from_django(use_case) if use_case else None + @strawberry_django.field @trace_resolver(name="get_datasets_by_use_case", attributes={"component": "usecase"}) def dataset_by_use_case(self, info: Info, use_case_id: str) -> list[TypeDataset]: diff --git a/api/services/metadata_export/adapter.py b/api/services/metadata_export/adapter.py index 9c8cfcda..f0215760 100644 --- a/api/services/metadata_export/adapter.py +++ b/api/services/metadata_export/adapter.py @@ -127,7 +127,7 @@ def dataset_to_record(dataset: Dataset) -> Dict[str, Any]: source = getattr(dataset, "source", None) org = dataset.organization user = dataset.user - landing = f"{_public_base()}/datasets/{dataset.slug}" + landing = f"{_public_base()}/datasets/{dataset.url_slug}" sectors = [vocab.concept(s.name, vocab.sector_uri(s.name)) for s in dataset.sectors.all()] geographies = [ diff --git a/api/services/platform_import_service.py b/api/services/platform_import_service.py index 0889cc9e..131090e2 100644 --- a/api/services/platform_import_service.py +++ b/api/services/platform_import_service.py @@ -13,7 +13,6 @@ import structlog from django.db import transaction -from django.utils.text import slugify from api.models import ( Dataset, @@ -84,19 +83,6 @@ def find_existing_import( return source.dataset if source else None -def _unique_dataset_slug(title: str) -> str: - """Pick a free slug up front. Dataset.save() retries on a slug collision, - but only a handful of times; platform titles ("Iris", "Wine Reviews") - repeat across organisations far more often than timestamped native ones, - so choose a free slug before saving rather than rely on the retry.""" - base = slugify(title)[:240] or "imported-dataset" - slug, counter = base, 2 - while Dataset.objects.filter(slug=slug).exists(): - slug = f"{base}-{counter}" - counter += 1 - return slug - - PLATFORM_LABELS = {"HUGGINGFACE": "Hugging Face", "KAGGLE": "Kaggle", "GITHUB": "GitHub"} @@ -191,7 +177,6 @@ def _create_import( display_title = (title or "").strip()[:300] or info.title dataset = Dataset.objects.create( title=display_title, - slug=_unique_dataset_slug(display_title), description=info.description, user=user, organization=organization, diff --git a/api/types/type_aimodel.py b/api/types/type_aimodel.py index e06baca6..efc824ec 100644 --- a/api/types/type_aimodel.py +++ b/api/types/type_aimodel.py @@ -115,6 +115,7 @@ class TypeAIModel(BaseType): """GraphQL type for AI Model.""" id: int + public_id: str name: str display_name: str version: Optional[str] @@ -142,6 +143,10 @@ class TypeAIModel(BaseType): updated_at: datetime last_tested_at: Optional[datetime] + @strawberry.field(description="Link part for the AI model page: title words + publicId.") + def url_slug(self) -> str: + return cast(AIModel, self).url_slug + @strawberry.field def endpoints(self, info: Info) -> List[TypeModelEndpoint]: """Get endpoints for this AI model. diff --git a/api/types/type_collaborative.py b/api/types/type_collaborative.py index e5bb1d13..0fd2d963 100644 --- a/api/types/type_collaborative.py +++ b/api/types/type_collaborative.py @@ -1,6 +1,6 @@ # mypy: disable-error-code="valid-type" -from typing import List, Optional +from typing import List, Optional, cast import strawberry import strawberry_django @@ -73,6 +73,10 @@ class TypeCollaborative(BaseType): description="URL of the platform where this collaborative is published" ) + @strawberry.field(description="Link part for the collaborative page: title words + publicId.") + def url_slug(self) -> str: + return cast(Collaborative, self).url_slug + @strawberry.field(description="Check if this collaborative is created by an individual user.") def is_individual_collaborative(self) -> bool: """Check if this collaborative is created by an individual user.""" diff --git a/api/types/type_dataset.py b/api/types/type_dataset.py index e8765930..d53dcb74 100644 --- a/api/types/type_dataset.py +++ b/api/types/type_dataset.py @@ -57,6 +57,7 @@ class TypeDataset(BaseType): title: str description: Optional[str] slug: str + public_id: str status: dataset_status dataset_type: dataset_type_enum organization: Optional["TypeOrganization"] @@ -66,6 +67,11 @@ class TypeDataset(BaseType): download_count: int user: Optional["TypeUser"] + @strawberry.field + def url_slug(self) -> str: + """Link part for the dataset page: title words + public_id.""" + return cast(Dataset, self).url_slug + @strawberry.field def source(self) -> Optional["TypeDatasetSource"]: """Provenance for datasets imported from a third-party platform (else null).""" diff --git a/api/types/type_publication.py b/api/types/type_publication.py index cfb40a51..b13dbd70 100644 --- a/api/types/type_publication.py +++ b/api/types/type_publication.py @@ -129,6 +129,7 @@ class TypePublication(BaseType): title: str description: Optional[str] slug: str + public_id: str status: publication_status authors: List[str] publication_date: Optional[date] @@ -141,6 +142,10 @@ class TypePublication(BaseType): user: Optional["TypeUser"] resource_type: Optional["TypeResourceType"] + @strawberry.field(description="Link part for the publication page: title words + publicId.") + def url_slug(self) -> str: + return cast(Publication, self).url_slug + @strawberry.field def sectors(self, info: Info) -> List["TypeSector"]: """Sectors tagged on this resource.""" diff --git a/api/types/type_usecase.py b/api/types/type_usecase.py index 60e8d6a0..6540daf4 100644 --- a/api/types/type_usecase.py +++ b/api/types/type_usecase.py @@ -1,6 +1,6 @@ # mypy: disable-error-code="valid-type" -from typing import List, Optional +from typing import List, Optional, cast import strawberry import strawberry_django @@ -72,6 +72,10 @@ class TypeUseCase(BaseType): description="URL of the platform where this use case is published" ) + @strawberry.field(description="Link part for the use case page: title words + publicId.") + def url_slug(self) -> str: + return cast(UseCase, self).url_slug + @strawberry.field(description="Check if this use case is created by an individual user.") def is_individual_usecase(self) -> bool: """Check if this use case is created by an individual user.""" diff --git a/api/utils/django_utils.py b/api/utils/django_utils.py index f9220439..2c38403f 100644 --- a/api/utils/django_utils.py +++ b/api/utils/django_utils.py @@ -1,46 +1,12 @@ -from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast -from django.core.exceptions import ValidationError -from django.db import IntegrityError, models, transaction +from django.db import models from django.db.models import Field, Model, QuerySet T = TypeVar("T", bound=models.Model) GraphQLType = TypeVar("GraphQLType") -def retry_on_slug_collision( - save_fn: Callable[[], None], - disambiguate: Callable[[int], None], - max_attempts: int = 5, -) -> None: - """Retry a model save that failed on a title/slug uniqueness collision. - - Auto-generated titles like "New dataset 17 Sep 2026 - 09:43:04" only have - one-second resolution, so two records created in the same second collide - on the unique slug (and, for models where title is also unique, on title - too). On each retry `disambiguate(attempt)` should mutate the instance's - title/slug to a new candidate (e.g. append " (n)") before `save_fn` is - called again. Each attempt runs in its own savepoint so a failed attempt - doesn't poison the caller's outer transaction. - - ponytail: catches ValidationError broadly (not just the title/slug keys) - so an unrelated validation failure retries pointlessly a few times before - surfacing the same error - upgrade to inspecting message_dict if that - noise ever matters. - """ - attempt = 0 - while True: - try: - with transaction.atomic(): - save_fn() - return - except (IntegrityError, ValidationError): - attempt += 1 - if attempt >= max_attempts: - raise - disambiguate(attempt) - - def get_graphql_type_fields_name(type_: Type[GraphQLType]) -> Dict[str, Any]: """Get field names from a GraphQL type.""" fields = type_.__dict__.get("__dataclass_fields__", {}) diff --git a/api/views/search_aimodel.py b/api/views/search_aimodel.py index dbf5ce6c..bb5a193c 100644 --- a/api/views/search_aimodel.py +++ b/api/views/search_aimodel.py @@ -20,6 +20,9 @@ class AIModelDocumentSerializer(serializers.ModelSerializer): """Serializer for AIModel document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + tags = serializers.ListField() sectors = serializers.ListField() geographies = serializers.ListField() @@ -77,6 +80,8 @@ class Meta: model = AIModel fields = [ "id", + "public_id", + "url_slug", "name", "display_name", "description", diff --git a/api/views/search_collaborative.py b/api/views/search_collaborative.py index 1bb7120c..a17a2344 100644 --- a/api/views/search_collaborative.py +++ b/api/views/search_collaborative.py @@ -55,6 +55,8 @@ def to_internal_value(self, data: Dict[str, Any]) -> Dict[str, Any]: class CollaborativeDocumentSerializer(serializers.ModelSerializer): + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) metadata = CollaborativeMetadataSerializer(many=True) tags = serializers.ListField() sectors = serializers.ListField(default=[]) @@ -107,6 +109,8 @@ class Meta: model = Collaborative fields = [ "id", + "public_id", + "url_slug", "title", "summary", "slug", diff --git a/api/views/search_dataset.py b/api/views/search_dataset.py index 5821803b..36ad6e7a 100644 --- a/api/views/search_dataset.py +++ b/api/views/search_dataset.py @@ -85,6 +85,8 @@ class DatasetDocumentSerializer(serializers.ModelSerializer): geographies = serializers.ListField() has_charts = serializers.BooleanField() slug = serializers.CharField() + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) download_count = serializers.IntegerField() trending_score = serializers.FloatField(required=False) is_individual_dataset = serializers.BooleanField() @@ -110,6 +112,8 @@ class Meta: "title", "description", "slug", + "public_id", + "url_slug", "created", "modified", "status", diff --git a/api/views/search_publication.py b/api/views/search_publication.py index 714a436f..21ee5159 100644 --- a/api/views/search_publication.py +++ b/api/views/search_publication.py @@ -25,6 +25,9 @@ class PublicationDocumentSerializer(serializers.ModelSerializer): """Serializer for the Publication search document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + resource_type = serializers.CharField(allow_blank=True) sectors = serializers.ListField() geographies = serializers.ListField() @@ -45,6 +48,8 @@ class Meta: model = Publication fields = [ "id", + "public_id", + "url_slug", "title", "description", "status", diff --git a/api/views/search_unified.py b/api/views/search_unified.py index 633cda3e..63707d0a 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -39,6 +39,8 @@ class UnifiedSearchResultSerializer(serializers.Serializer): title = serializers.CharField() description = serializers.CharField() slug = serializers.CharField(required=False) + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) created = serializers.DateTimeField(required=False) modified = serializers.DateTimeField(required=False) created_at = serializers.DateTimeField(required=False) diff --git a/api/views/search_usecase.py b/api/views/search_usecase.py index a0edeaf7..a6e9a639 100644 --- a/api/views/search_usecase.py +++ b/api/views/search_usecase.py @@ -61,6 +61,9 @@ def to_internal_value(self, data: Dict[str, Any]) -> Dict[str, Any]: class UseCaseDocumentSerializer(serializers.ModelSerializer): """Serializer for UseCase document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + metadata = UseCaseMetadataSerializer(many=True) tags = serializers.ListField() logo = serializers.CharField() @@ -109,6 +112,8 @@ class Meta: model = UseCase fields = [ "id", + "public_id", + "url_slug", "title", "summary", "slug", diff --git a/authorization/permissions.py b/authorization/permissions.py index fb27f87f..d38e6ede 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -7,6 +7,7 @@ from api.models import Dataset, Event, Organization, Publication from api.utils.enums import EventStatus, PublicationStatus +from api.utils.public_links import parse_public_id from authorization.models import DatasetPermission, OrganizationMembership, Role # Roles that may publish/unpublish and edit an org-owned publication, by name. @@ -581,6 +582,14 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True +def find_publication(publication_id: Any, public_id: Optional[str]) -> Optional[Publication]: + """The publication for a UUID, or for a public link (``urlSlug`` or ``publicId``).""" + if publication_id: + return Publication.objects.filter(id=publication_id).first() + parsed = parse_public_id(public_id) + return Publication.objects.filter(public_id=parsed).first() if parsed else None + + class AllowPublishedPublications(BasePermission): # type: ignore[misc] """Read gate for a single publication — mirrors AllowPublishedDatasets. @@ -593,11 +602,11 @@ class AllowPublishedPublications(BasePermission): # type: ignore[misc] def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: request = info.context publication_id = kwargs.get("publication_id") + public_id = kwargs.get("public_id") - if publication_id: - try: - publication = Publication.objects.get(id=publication_id) - except Publication.DoesNotExist: + if publication_id or public_id: + publication = find_publication(publication_id, public_id) + if publication is None: return True # Let the resolver return a clean not-found. if publication.status == PublicationStatus.PUBLISHED.value: diff --git a/search/documents/aimodel_document.py b/search/documents/aimodel_document.py index 6cd6a1a7..400da070 100644 --- a/search/documents/aimodel_document.py +++ b/search/documents/aimodel_document.py @@ -24,6 +24,8 @@ class AIModelDocument(Document): """Elasticsearch document for AIModel.""" # Basic fields with analyzers + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") name = fields.TextField( analyzer=ngram_analyser, fields=title_subfields(), diff --git a/search/documents/collaborative_document.py b/search/documents/collaborative_document.py index 1fedaca3..3ab5495f 100644 --- a/search/documents/collaborative_document.py +++ b/search/documents/collaborative_document.py @@ -32,6 +32,8 @@ class CollaborativeDocument(Document): """Elasticsearch document for Collaborative model.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") metadata = fields.NestedField( properties={ "value": KeywordField(multi=True), diff --git a/search/documents/dataset_document.py b/search/documents/dataset_document.py index 757920c2..e144419e 100644 --- a/search/documents/dataset_document.py +++ b/search/documents/dataset_document.py @@ -57,6 +57,8 @@ class DatasetDocument(Document): status = fields.KeywordField() slug = fields.KeywordField() # Add slug field + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") tags = fields.TextField( attr="tags_indexing", diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py index 3c1e6e7f..e5fd45a7 100644 --- a/search/documents/publication_document.py +++ b/search/documents/publication_document.py @@ -29,6 +29,8 @@ class PublicationDocument(Document): """Elasticsearch document for a published Resource.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") title = fields.TextField( analyzer=ngram_analyser, fields=title_subfields(), diff --git a/search/documents/usecase_document.py b/search/documents/usecase_document.py index 9849d753..9e292d65 100644 --- a/search/documents/usecase_document.py +++ b/search/documents/usecase_document.py @@ -24,6 +24,8 @@ class UseCaseDocument(Document): """Elasticsearch document for UseCase model.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") metadata = fields.NestedField( properties={ "value": KeywordField(multi=True), diff --git a/tests/schema/test_dataset_public_links.py b/tests/schema/test_dataset_public_links.py new file mode 100644 index 00000000..a9a59bca --- /dev/null +++ b/tests/schema/test_dataset_public_links.py @@ -0,0 +1,98 @@ +import types + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Dataset +from api.schema.schema import schema +from api.services.metadata_export.adapter import dataset_to_record +from api.utils.enums import DatasetStatus +from authorization.models import User + +QUERY = """ +query($id: UUID, $pid: String) { + getDataset(datasetId: $id, publicId: $pid) { id title publicId urlSlug } +} +""" + + +def run(query, user, variables): + ctx = types.SimpleNamespace(user=user, context={}) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def published(db): + return Dataset.objects.create(title="City Air Quality Index", status=DatasetStatus.PUBLISHED) + + +@pytest.fixture +def draft(db): + return Dataset.objects.create(title="Secret draft", status=DatasetStatus.DRAFT) + + +def test_new_dataset_gets_public_id_and_link(published) -> None: + assert len(published.public_id) == 12 + assert published.url_slug == f"city-air-quality-index-{published.public_id}" + + +def test_rename_keeps_public_id(published) -> None: + old = published.public_id + published.title = "Air Quality Index for Cities" + published.save() + published.refresh_from_db() + assert published.public_id == old + assert published.url_slug == f"air-quality-index-for-cities-{old}" + + +def test_get_by_public_id_anonymous(published) -> None: + for value in [published.public_id, published.url_slug, f"old-words-{published.public_id}"]: + res = run(QUERY, AnonymousUser(), {"pid": value}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["id"] == str(published.id) + assert res.data["getDataset"]["urlSlug"] == published.url_slug + + +def test_get_by_uuid_still_works(published) -> None: + res = run(QUERY, AnonymousUser(), {"id": str(published.id)}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["publicId"] == published.public_id + + +def test_unknown_or_bad_public_id_gives_null(published) -> None: + user = User.objects.create(username="u1", keycloak_id="u1") + for value in ["aaaaaaaaaaaa", "not-a-link", str(published.id)]: + res = run(QUERY, user, {"pid": value}) + assert res.errors is None, res.errors + assert res.data["getDataset"] is None + + +def test_draft_by_public_id_needs_login(draft) -> None: + res = run(QUERY, AnonymousUser(), {"pid": draft.public_id}) + assert res.errors + assert res.data is None or res.data["getDataset"] is None + + +def test_draft_by_public_id_other_user_denied(draft) -> None: + stranger = User.objects.create(username="u2", keycloak_id="u2") + res = run(QUERY, stranger, {"pid": draft.public_id}) + assert res.errors + + +def test_draft_by_public_id_superuser(draft) -> None: + admin = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + res = run(QUERY, admin, {"pid": draft.public_id}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["publicId"] == draft.public_id + + +def test_export_landing_link_uses_url_slug(published) -> None: + record = dataset_to_record(published) + assert record["landing_page"].endswith(f"/datasets/{published.url_slug}") + + +def test_hindi_title_link_is_just_the_id(db) -> None: + ds = Dataset.objects.create(title="जनगणना डेटा", status=DatasetStatus.PUBLISHED) + assert ds.url_slug == ds.public_id + res = run(QUERY, AnonymousUser(), {"pid": ds.url_slug}) + assert res.data["getDataset"]["id"] == str(ds.id) diff --git a/tests/schema/test_module_public_links.py b/tests/schema/test_module_public_links.py new file mode 100644 index 00000000..5c1e642b --- /dev/null +++ b/tests/schema/test_module_public_links.py @@ -0,0 +1,179 @@ +import types + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import AIModel, Collaborative, Publication, UseCase +from api.schema.schema import schema +from api.utils.enums import CollaborativeStatus, PublicationStatus, UseCaseStatus +from authorization.models import User + + +def run(query, variables, user=None): + ctx = types.SimpleNamespace(user=user or AnonymousUser(), context={}) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +def make_model(owner, **kw): + data = dict( + name="sarvam-translate", + display_name="Sarvam Translate", + description="d", + model_type="TRANSLATION", + provider="OPENAI", + user=owner, + ) + data.update(kw) + return AIModel.objects.create(**data) + + +# --- every module: id, link, same title twice, rename keeps id --------------- + + +@pytest.mark.parametrize("kind", ["usecase", "collaborative", "publication", "aimodel"]) +def test_ids_links_and_duplicate_titles(kind, owner) -> None: + def make(title): + if kind == "usecase": + return UseCase.objects.create(title=title, user=owner) + if kind == "collaborative": + return Collaborative.objects.create(title=title, user=owner) + if kind == "publication": + return Publication.objects.create(title=title, user=owner) + return make_model(owner, display_name=title, name=f"m-{AIModel.objects.count()}") + + a = make("Air Quality Study") + b = make("Air Quality Study") + assert len(a.public_id) == 12 and a.public_id != b.public_id + assert a.url_slug == f"air-quality-study-{a.public_id}" + assert a.url_slug != b.url_slug + + old = a.public_id + if kind == "aimodel": + a.display_name = "Air Quality Study v2" + else: + a.title = "Air Quality Study v2" + a.save() + a.refresh_from_db() + assert a.public_id == old + assert a.url_slug == f"air-quality-study-v2-{old}" + + +def test_long_collaborative_title_saves(owner) -> None: + # full_clean checks slug length and format; the cut must not leave a trailing hyphen. + title = "District wise monthly rainfall and temperature readings for all states 1990" + c = Collaborative.objects.create(title=title, user=owner) + assert len(c.slug) <= 75 and not c.slug.endswith("-") + + +# --- use cases ----------------------------------------------------------------- + +UC = "query($s: String!) { useCaseBySlug(slug: $s) { id publicId urlSlug } }" + + +def test_use_case_by_link(owner) -> None: + uc = UseCase.objects.create(title="Flood Maps", user=owner, status=UseCaseStatus.PUBLISHED) + for value in [uc.url_slug, uc.public_id, f"old-words-{uc.public_id}"]: + res = run(UC, {"s": value}) + assert res.errors is None, res.errors + assert res.data["useCaseBySlug"]["id"] == str(uc.id) + assert res.data["useCaseBySlug"]["urlSlug"] == uc.url_slug + + +def test_use_case_by_link_hides_drafts_and_plain_words(owner) -> None: + draft = UseCase.objects.create(title="Secret", user=owner) + pub = UseCase.objects.create(title="Open One", user=owner, status=UseCaseStatus.PUBLISHED) + for value in [draft.url_slug, pub.slug, "aaaaaaaaaaaa", str(pub.id)]: + res = run(UC, {"s": value}) + assert res.errors is None, res.errors + assert res.data["useCaseBySlug"] is None + + +# --- collaboratives ------------------------------------------------------------ + +CB = "query($s: String!) { collaborativeBySlug(slug: $s) { id publicId urlSlug } }" + + +def test_collaborative_by_link_and_old_slug(owner) -> None: + c = Collaborative.objects.create( + title="Clean Rivers", user=owner, status=CollaborativeStatus.PUBLISHED + ) + for value in [c.url_slug, c.public_id, f"old-words-{c.public_id}", c.slug]: + res = run(CB, {"s": value}) + assert res.errors is None, res.errors + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + + +def test_collaborative_old_slug_with_duplicates_gives_oldest(owner) -> None: + first = Collaborative.objects.create(title="Clean Rivers", user=owner) + Collaborative.objects.create(title="Clean Rivers", user=owner) + res = run(CB, {"s": "clean-rivers"}) + assert res.data["collaborativeBySlug"]["id"] == str(first.id) + + +def test_collaborative_unknown_link_errors(owner) -> None: + res = run(CB, {"s": "nothing-aaaaaaaaaaaa"}) + assert res.errors + + +# --- publications -------------------------------------------------------------- + +PB = """query($id: UUID, $p: String) { + getPublication(publicationId: $id, publicId: $p) { id publicId urlSlug } +}""" + + +def test_publication_by_public_id(owner) -> None: + p = Publication.objects.create( + title="Budget Brief", user=owner, status=PublicationStatus.PUBLISHED + ) + for value in [p.url_slug, p.public_id]: + res = run(PB, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getPublication"]["id"] == str(p.id) + res = run(PB, {"id": str(p.id)}) + assert res.data["getPublication"]["publicId"] == p.public_id + + +def test_publication_draft_by_public_id(owner) -> None: + p = Publication.objects.create(title="Draft Brief", user=owner) + assert run(PB, {"p": p.public_id}).errors # anonymous refused + stranger = User.objects.create(username="s", keycloak_id="s") + assert run(PB, {"p": p.public_id}, stranger).errors + res = run(PB, {"p": p.public_id}, owner) + assert res.errors is None and res.data["getPublication"]["id"] == str(p.id) + + +def test_publication_unknown_gives_null(owner) -> None: + for value in ["aaaaaaaaaaaa", "no-id-here"]: + res = run(PB, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getPublication"] is None + + +# --- AI models ----------------------------------------------------------------- + +AM = """query($id: Int, $p: String) { + getAiModel(modelId: $id, publicId: $p) { id publicId urlSlug } +}""" + + +def test_ai_model_by_public_id(owner) -> None: + m = make_model(owner, is_public=True, is_active=True) + for value in [m.url_slug, m.public_id]: + res = run(AM, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getAiModel"]["id"] == m.id + assert run(AM, {"id": m.id}).data["getAiModel"]["publicId"] == m.public_id + + +def test_ai_model_private_and_unknown(owner) -> None: + m = make_model(owner, is_public=False, is_active=True) + assert run(AM, {"p": m.public_id}).data["getAiModel"] is None + assert run(AM, {"p": m.public_id}, owner).data["getAiModel"]["id"] == m.id + assert run(AM, {"p": "aaaaaaaaaaaa"}).data["getAiModel"] is None + assert run(AM, {"p": "words-only"}).data["getAiModel"] is None diff --git a/tests/test_public_links.py b/tests/test_public_links.py new file mode 100644 index 00000000..9b287f3d --- /dev/null +++ b/tests/test_public_links.py @@ -0,0 +1,157 @@ +import uuid + +import pytest +from django.db import IntegrityError + +from api.utils.public_links import ( + PUBLIC_ID_LENGTH, + build_url_slug, + generate_public_id, + parse_public_id, + save_with_public_id, +) + + +class Record: + public_id = "" + + +def test_generate_public_id_format_and_randomness() -> None: + ids = {generate_public_id() for _ in range(20000)} + assert len(ids) == 20000 + for value in list(ids)[:500]: + assert len(value) == PUBLIC_ID_LENGTH + assert value.isascii() and (value.islower() or value.isdigit()) and value.isalnum() + + +def test_save_assigns_id_when_missing() -> None: + record = Record() + calls = [] + save_with_public_id(record, lambda: calls.append(record.public_id)) + assert len(calls) == 1 and calls[0] == record.public_id + assert parse_public_id(record.public_id) == record.public_id + + +def test_save_keeps_existing_id() -> None: + record = Record() + record.public_id = "k3x9m2qa7b1z" + save_with_public_id(record, lambda: None) + assert record.public_id == "k3x9m2qa7b1z" + + +def test_save_retries_with_new_id_on_clash() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(record.public_id) + if len(tried) < 3: + raise IntegrityError("UNIQUE constraint failed: dataset.public_id") + + save_with_public_id(record, save) + assert len(tried) == 3 and len(set(tried)) == 3 + assert record.public_id == tried[-1] + + +def test_save_postgres_style_error_is_retried() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(record.public_id) + if len(tried) == 1: + raise IntegrityError( + 'duplicate key value violates unique constraint "dataset_public_id_key"\n' + "DETAIL: Key (public_id)=(abc) already exists." + ) + + save_with_public_id(record, save) + assert len(tried) == 2 + + +def test_save_other_unique_errors_are_not_retried() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(1) + raise IntegrityError("UNIQUE constraint failed: dataset.slug") + + with pytest.raises(IntegrityError): + save_with_public_id(record, save) + assert len(tried) == 1 + + +def test_save_gives_up_after_max_attempts() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(1) + raise IntegrityError("UNIQUE constraint failed: dataset.public_id") + + with pytest.raises(IntegrityError): + save_with_public_id(record, save, max_attempts=4) + assert len(tried) == 4 + + +@pytest.mark.parametrize( + "title, expected", + [ + ("City Air Quality Index", "city-air-quality-index-k3x9m2qa7b1z"), + (" Rainfall: 2020–2024 (Daily)! ", "rainfall-20202024-daily-k3x9m2qa7b1z"), + ("जनगणना डेटा 2011", "2011-k3x9m2qa7b1z"), + ("जनगणना डेटा", "k3x9m2qa7b1z"), + ("", "k3x9m2qa7b1z"), + (None, "k3x9m2qa7b1z"), + ("!!! ???", "k3x9m2qa7b1z"), + ], +) +def test_build_url_slug(title, expected) -> None: + assert build_url_slug(title, "k3x9m2qa7b1z") == expected + + +def test_build_url_slug_cuts_long_titles_between_words() -> None: + title = "District wise monthly rainfall and temperature readings for all states of India 1990 to 2024" + assert ( + build_url_slug(title, "k3x9m2qa7b1z") + == "district-wise-monthly-rainfall-and-temperature-k3x9m2qa7b1z" + ) + + +def test_build_url_slug_exact_limit_and_one_long_word() -> None: + exact = "a" * 20 + " " + "b" * 29 # 50 chars once slugified + assert build_url_slug(exact, "k3x9m2qa7b1z") == f"{'a' * 20}-{'b' * 29}-k3x9m2qa7b1z" + long_word = "x" * 80 + assert build_url_slug(long_word, "k3x9m2qa7b1z") == f"{'x' * 50}-k3x9m2qa7b1z" + + +@pytest.mark.parametrize( + "link, expected", + [ + ("city-air-quality-index-k3x9m2qa7b1z", "k3x9m2qa7b1z"), + ("k3x9m2qa7b1z", "k3x9m2qa7b1z"), + ("/city-air-quality-index-K3X9M2QA7B1Z/", "k3x9m2qa7b1z"), + ("old-words-that-changed-k3x9m2qa7b1z", "k3x9m2qa7b1z"), + (str(uuid.UUID("3f2a9c1e-7b4d-4e2a-9c1e-a1b2c3d4e5f6")), None), + ("3f2a9c1e7b4d4e2a9c1ea1b2c3d4e5f6", None), + ("123", None), + ("city-air-quality-index", None), + ("city-air-quality-k3x9m2qa7b1", None), + ("city-air-quality-k3x9m2qa7b1z9", None), + ("city-air-quality-k3x9m2qa7b1_", None), + ("city-air-quality-k3x9m2qa7b1z\n/", None), + ("k3x9m2qa7b1٣", None), + ("k3x9m2qa7b1é", None), + ("", None), + (None, None), + ], +) +def test_parse_public_id(link, expected) -> None: + assert parse_public_id(link) == expected + + +def test_build_then_parse_round_trip() -> None: + for title in ["City Air Quality Index", "जनगणना डेटा", "", "x" * 300]: + public_id = generate_public_id() + assert parse_public_id(build_url_slug(title, public_id)) == public_id diff --git a/tests/test_publication_models.py b/tests/test_publication_models.py index 6cc2209c..7e43d686 100644 --- a/tests/test_publication_models.py +++ b/tests/test_publication_models.py @@ -26,20 +26,23 @@ def org(db): @pytest.mark.django_db class TestPublicationSlug: - def test_two_same_title_publications_get_distinct_slugs(self, user): + def test_two_same_title_publications_get_distinct_links(self, user): + # Slugs may repeat; each publication is found by its own public_id. first = Publication.objects.create(title="Annual Report", user=user) second = Publication.objects.create(title="Annual Report", user=user) - assert first.slug == "annual-report" - assert second.slug == "annual-report-1" - assert first.slug != second.slug + assert first.slug == second.slug == "annual-report" + assert first.public_id != second.public_id + assert first.url_slug == f"annual-report-{first.public_id}" + assert first.url_slug != second.url_slug - def test_a_blank_title_still_gets_a_unique_slug(self, user): + def test_a_blank_title_still_gets_a_link(self, user): first = Publication.objects.create(title="", user=user) second = Publication.objects.create(title=" ", user=user) - assert first.slug == "untitled" - assert second.slug == "untitled-1" + assert first.slug == second.slug == "untitled" + assert first.url_slug == first.public_id + assert first.public_id != second.public_id def test_a_unicode_title_slugs_without_crashing(self, user): publication = Publication.objects.create(title="Report — Résumé 2024", user=user) diff --git a/tests/test_slug_collision.py b/tests/test_slug_collision.py index c94263bf..b2cc9440 100644 --- a/tests/test_slug_collision.py +++ b/tests/test_slug_collision.py @@ -1,17 +1,12 @@ """Regression test for the same-second slug/title collision. -Root cause: auto-generated titles (e.g. "New dataset 17 Sep 2026 - 09:43:04") -have one-second resolution, and Dataset.slug / UseCase.title+slug / -Collaborative.title+slug are unique columns. Two creates landing in the same -second (routine under concurrent CI load - e.g. `-n 3`) previously raised an -IntegrityError/ValidationError that the create mutation surfaced as a bare -`success: False` with no usable error, which is what made -test_prv_006_org_create_dataset and test_prv_007_org_create_usecase in -CivicDataSpace-test fail under concurrent load (DataSpaceBackend #199). - -Model.save() now retries with a disambiguated title/slug on collision -(api/utils/django_utils.py:retry_on_slug_collision), so this same-second -create no longer fails. +Auto-generated titles (e.g. "New dataset 17 Sep 2026 - 09:43:04") have +one-second resolution, so two creates in the same second get the same title. +Titles and slugs used to be unique, so the second create failed under +concurrent load (DataSpaceBackend #199). + +Titles and slugs no longer need to be unique: each record gets its own +public_id, and links are found by that id. Two records may share a title. """ import pytest @@ -22,12 +17,16 @@ @pytest.mark.django_db def test_dataset_same_second_title_does_not_collide(): + # Dataset slugs are no longer unique; each dataset is found by its own + # public_id, so two datasets may share a title and slug. same_title = "New dataset 18 Sep 2026 - 09:43:04" first = Dataset.objects.create(title=same_title) second = Dataset.objects.create(title=same_title) - assert first.slug != second.slug - assert second.slug.startswith(first.slug) + assert first.slug == second.slug + assert first.public_id and second.public_id + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug @pytest.mark.django_db @@ -38,9 +37,9 @@ def test_usecase_same_second_title_does_not_collide(): first = UseCase.objects.create(title=same_title, user=user) second = UseCase.objects.create(title=same_title, user=user) - assert first.title == same_title - assert second.title != same_title - assert second.slug != first.slug + assert first.title == second.title == same_title + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug @pytest.mark.django_db @@ -51,6 +50,6 @@ def test_collaborative_same_second_title_does_not_collide(): first = Collaborative.objects.create(title=same_title, user=user) second = Collaborative.objects.create(title=same_title, user=user) - assert first.title == same_title - assert second.title != same_title - assert second.slug != first.slug + assert first.title == second.title == same_title + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug From 3c964ba04bd3ca3598d727e68778d07d9ee67253 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 3 Nov 2026 01:11:16 +0530 Subject: [PATCH 39/55] feat(api): take publicId on every public page query --- api/schema/collaborative_schema.py | 24 +++++++++++++----------- api/schema/event_schema.py | 16 ++++++++-------- api/schema/usecase_schema.py | 18 ++++++++---------- tests/schema/test_event_schema.py | 8 ++++---- tests/schema/test_module_public_links.py | 13 ++++++++++++- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 58a800d9..a1eca2d2 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -293,21 +293,23 @@ def contributors_by_collaborative(self, info: Info, collaborative_id: str) -> li name="get_collaborative_by_slug", attributes={"component": "collaborative"}, ) - def collaborative_by_slug(self, info: Info, slug: str) -> TypeCollaborative: - """Collaborative for its page, found by its link (``urlSlug``). + def collaborative_by_slug( + self, info: Info, public_id: Optional[str] = None, slug: Optional[str] = None + ) -> TypeCollaborative: + """Collaborative for its page, found by its link. - Only the publicId at the end of the link is used, so a link with old - title words still works. Links shared before public ids existed carry - the old slug, so a value without a matching id falls back to the slug. + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. ``slug`` is for links and subdomains made before public ids + existed: the id is tried first, then the old slug. """ - public_id = parse_public_id(slug) - collaborative = ( - Collaborative.objects.filter(public_id=public_id).first() if public_id else None - ) - if collaborative is None: + value = public_id if public_id is not None else slug + parsed = parse_public_id(value) + collaborative = Collaborative.objects.filter(public_id=parsed).first() if parsed else None + if collaborative is None and public_id is None and slug: collaborative = Collaborative.objects.filter(slug=slug).order_by("created").first() if collaborative is None: - raise ValueError(f"Collaborative with slug {slug} does not exist.") + raise ValueError(f"Collaborative {value} does not exist.") return TypeCollaborative.from_django(collaborative) diff --git a/api/schema/event_schema.py b/api/schema/event_schema.py index 77d84b57..95a85034 100644 --- a/api/schema/event_schema.py +++ b/api/schema/event_schema.py @@ -166,18 +166,18 @@ def published_events( @strawberry.field @trace_resolver(name="event_by_slug", attributes={"component": "event"}) - def event_by_slug(self, info: Info, slug: str) -> Optional[TypeEvent]: - """Published event for a public page, found by its link (``urlSlug``). + def event_by_slug(self, info: Info, public_id: str) -> Optional[TypeEvent]: + """Published event for a public page, found by its link. - Only the publicId at the end of the link is used, so a link with old - title words still works. Drafts, unknown ids and links without an id - return null. + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. Drafts, unknown ids and links without an id return null. """ - public_id = parse_public_id(slug) - if public_id is None: + parsed = parse_public_id(public_id) + if parsed is None: return None event = ( - Event.objects.filter(public_id=public_id, status=EventStatus.PUBLISHED) + Event.objects.filter(public_id=parsed, status=EventStatus.PUBLISHED) .select_related("organization", "user") .first() ) diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index 6cf3a22e..7272b4aa 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -275,19 +275,17 @@ def published_use_cases( @strawberry_django.field @trace_resolver(name="get_use_case_by_slug", attributes={"component": "usecase"}) - def use_case_by_slug(self, info: Info, slug: str) -> Optional[TypeUseCase]: - """Published use case for a public page, found by its link (``urlSlug``). + def use_case_by_slug(self, info: Info, public_id: str) -> Optional[TypeUseCase]: + """Published use case for a public page, found by its link. - Only the publicId at the end of the link is used, so a link with old - title words still works. Drafts, unknown ids and links without an id - return null. + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. Drafts, unknown ids and links without an id return null. """ - public_id = parse_public_id(slug) - if public_id is None: + parsed = parse_public_id(public_id) + if parsed is None: return None - use_case = UseCase.objects.filter( - public_id=public_id, status=UseCaseStatus.PUBLISHED - ).first() + use_case = UseCase.objects.filter(public_id=parsed, status=UseCaseStatus.PUBLISHED).first() return TypeUseCase.from_django(use_case) if use_case else None @strawberry_django.field diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py index 1b718d71..8590da57 100644 --- a/tests/schema/test_event_schema.py +++ b/tests/schema/test_event_schema.py @@ -173,8 +173,8 @@ def _make_event(user, org, **overrides): """ BY_SLUG = """ -query BySlug($slug: String!) { - eventBySlug(slug: $slug) { name urlSlug status lumaEmbed { src } } +query BySlug($publicId: String!) { + eventBySlug(publicId: $publicId) { name urlSlug status lumaEmbed { src } } } """ @@ -365,7 +365,7 @@ def test_published_listing_can_be_limited_to_one_organization( def test_event_by_slug_returns_a_published_event(self, org_a_admin, org_a): event = _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) - result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.url_slug}) + result = run(BY_SLUG, ctx(AnonymousUser()), {"publicId": event.url_slug}) assert result.errors is None assert result.data["eventBySlug"]["urlSlug"] == event.url_slug @@ -374,7 +374,7 @@ def test_event_by_slug_returns_a_published_event(self, org_a_admin, org_a): def test_event_by_slug_hides_drafts(self, org_a_admin, org_a): event = _make_event(org_a_admin, org_a, name="Draft night") - result = run(BY_SLUG, ctx(AnonymousUser()), {"slug": event.url_slug}) + result = run(BY_SLUG, ctx(AnonymousUser()), {"publicId": event.url_slug}) assert result.errors is None assert result.data["eventBySlug"] is None diff --git a/tests/schema/test_module_public_links.py b/tests/schema/test_module_public_links.py index 5c1e642b..05cc2dd2 100644 --- a/tests/schema/test_module_public_links.py +++ b/tests/schema/test_module_public_links.py @@ -72,7 +72,7 @@ def test_long_collaborative_title_saves(owner) -> None: # --- use cases ----------------------------------------------------------------- -UC = "query($s: String!) { useCaseBySlug(slug: $s) { id publicId urlSlug } }" +UC = "query($s: String!) { useCaseBySlug(publicId: $s) { id publicId urlSlug } }" def test_use_case_by_link(owner) -> None: @@ -96,6 +96,7 @@ def test_use_case_by_link_hides_drafts_and_plain_words(owner) -> None: # --- collaboratives ------------------------------------------------------------ CB = "query($s: String!) { collaborativeBySlug(slug: $s) { id publicId urlSlug } }" +CB_ID = "query($s: String!) { collaborativeBySlug(publicId: $s) { id publicId urlSlug } }" def test_collaborative_by_link_and_old_slug(owner) -> None: @@ -106,6 +107,16 @@ def test_collaborative_by_link_and_old_slug(owner) -> None: res = run(CB, {"s": value}) assert res.errors is None, res.errors assert res.data["collaborativeBySlug"]["id"] == str(c.id) + for value in [c.url_slug, c.public_id, f"old-words-{c.public_id}"]: + res = run(CB_ID, {"s": value}) + assert res.errors is None, res.errors + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + + +def test_collaborative_public_id_does_not_fall_back_to_slug(owner) -> None: + c = Collaborative.objects.create(title="Clean Rivers", user=owner) + res = run(CB_ID, {"s": c.slug}) + assert res.errors def test_collaborative_old_slug_with_duplicates_gives_oldest(owner) -> None: From 8c13f5997c726eb080294dfaddc79e3320d22862 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 3 Nov 2026 01:55:56 +0530 Subject: [PATCH 40/55] fix(api): keep collaborative slugs unique, they are also the subdomain --- api/migrations/0058_public_ids.py | 26 +++------------- api/models/Collaborative.py | 39 ++++++++++++++++++++---- tests/schema/test_module_public_links.py | 31 +++++++++++++++++-- tests/test_slug_collision.py | 2 ++ 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/api/migrations/0058_public_ids.py b/api/migrations/0058_public_ids.py index 7b5c26e1..b21e6cde 100644 --- a/api/migrations/0058_public_ids.py +++ b/api/migrations/0058_public_ids.py @@ -1,4 +1,3 @@ -import django.core.validators from django.db import migrations, models MODELS = ["aimodel", "collaborative", "dataset", "publication", "usecase"] @@ -7,9 +6,10 @@ class Migration(migrations.Migration): """Add public_id (nullable for now) to every public module. - Also drops the unique rule on slugs and on use case / collaborative titles: - links use public_id, so titles may repeat. Filled for existing rows in - 0057, made unique and required in 0058. + Also drops the unique rule on dataset, publication and use case slugs and + on use case / collaborative titles: links use public_id, so titles may + repeat. Collaborative slugs stay unique (the slug is also the subdomain). + Filled for existing rows in 0059, made unique and required in 0060. """ dependencies = [ @@ -25,24 +25,6 @@ class Migration(migrations.Migration): ) for model in MODELS ], - migrations.AlterField( - model_name="collaborative", - name="slug", - field=models.SlugField( - blank=True, - max_length=75, - null=True, - validators=[ - django.core.validators.RegexValidator( - message=( - "Slug must be lowercase and contain only alphanumeric " - "characters and hyphens." - ), - regex="^[a-z0-9]+(?:-[a-z0-9]+)*$", - ) - ], - ), - ), migrations.AlterField( model_name="collaborative", name="title", diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 345a4b15..99020d71 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, cast from django.core.validators import RegexValidator -from django.db import models +from django.db import IntegrityError, models, transaction from django.utils.text import slugify if TYPE_CHECKING: @@ -12,7 +12,12 @@ from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path -from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id +from api.utils.public_links import ( + PUBLIC_ID_LENGTH, + build_url_slug, + generate_public_id, + save_with_public_id, +) from api.utils.storage import public_storage slug_validator = RegexValidator( @@ -21,6 +26,11 @@ ) +def _slug_with_suffix(base: str) -> str: + """``base`` plus 4 random characters, kept within the 75-character limit.""" + return f"{base[:70].rstrip('-')}-{generate_public_id()[:4]}" + + class Collaborative(models.Model): id = models.AutoField(primary_key=True) title = models.CharField(max_length=200, blank=True, null=True) @@ -43,8 +53,10 @@ class Collaborative(models.Model): modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - # Kept for old links and older readers; no longer unique. Links use public_id. - slug = models.SlugField(max_length=75, null=True, blank=True, validators=[slug_validator]) + # Unique: the slug is also the collaborative's subdomain. Page links use public_id. + slug = models.SlugField( + max_length=75, null=True, blank=True, unique=True, validators=[slug_validator] + ) public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( @@ -77,15 +89,30 @@ class Collaborative(models.Model): platform_url = models.URLField(blank=True, null=True) def save(self, *args: Any, **kwargs: Any) -> None: + base = "" if self.title and not self.slug: - self.slug = slugify(cast(str, self.title))[:75].strip("-") + base = slugify(cast(str, self.title))[:75].strip("-") + if base: + taken = Collaborative.objects.filter(slug=base).exclude(pk=self.pk).exists() + self.slug = _slug_with_suffix(base) if taken else base def do_save() -> None: # public_id clashes are caught by the database and retried, not here. self.full_clean(exclude=["public_id"]) super(Collaborative, self).save(*args, **kwargs) - save_with_public_id(self, do_save) + # Titles may repeat but slugs may not. A slug made from the title that + # another save took in the meantime gets a new suffix; a slug the owner + # typed is never changed (full_clean refuses it if taken). + for attempt in range(3): + try: + with transaction.atomic(): + save_with_public_id(self, do_save) + return + except IntegrityError as exc: + if not base or "slug" not in str(exc) or attempt == 2: + raise + self.slug = _slug_with_suffix(base) @property def url_slug(self) -> str: diff --git a/tests/schema/test_module_public_links.py b/tests/schema/test_module_public_links.py index 05cc2dd2..528191a8 100644 --- a/tests/schema/test_module_public_links.py +++ b/tests/schema/test_module_public_links.py @@ -2,6 +2,8 @@ import pytest from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import ValidationError +from django.db.models import QuerySet from api.models import AIModel, Collaborative, Publication, UseCase from api.schema.schema import schema @@ -119,11 +121,34 @@ def test_collaborative_public_id_does_not_fall_back_to_slug(owner) -> None: assert res.errors -def test_collaborative_old_slug_with_duplicates_gives_oldest(owner) -> None: +def test_collaborative_same_title_gets_its_own_slug(owner) -> None: + # The slug is also the subdomain, so a repeated title gets a suffix. first = Collaborative.objects.create(title="Clean Rivers", user=owner) + second = Collaborative.objects.create(title="Clean Rivers", user=owner) + assert first.slug == "clean-rivers" + assert second.slug.startswith("clean-rivers-") and len(second.slug) == len("clean-rivers") + 5 + for c in [first, second]: + res = run(CB, {"s": c.slug}) + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + + +def test_collaborative_taken_slug_is_refused(owner) -> None: Collaborative.objects.create(title="Clean Rivers", user=owner) - res = run(CB, {"s": "clean-rivers"}) - assert res.data["collaborativeBySlug"]["id"] == str(first.id) + other = Collaborative.objects.create(title="Something else", user=owner) + other.slug = "clean-rivers" + with pytest.raises(ValidationError): + other.save() + + +def test_collaborative_slug_race_gets_a_new_suffix(owner, monkeypatch) -> None: + # Another save takes the slug between the check and the insert. + Collaborative.objects.create(title="Clean Rivers", user=owner) + monkeypatch.setattr(Collaborative, "full_clean", lambda self, **kw: None) + real_exists = QuerySet.exists + monkeypatch.setattr(QuerySet, "exists", lambda self: False) + c = Collaborative.objects.create(title="Clean Rivers", user=owner) + monkeypatch.setattr(QuerySet, "exists", real_exists) + assert c.slug.startswith("clean-rivers-") def test_collaborative_unknown_link_errors(owner) -> None: diff --git a/tests/test_slug_collision.py b/tests/test_slug_collision.py index b2cc9440..a4ae31ea 100644 --- a/tests/test_slug_collision.py +++ b/tests/test_slug_collision.py @@ -51,5 +51,7 @@ def test_collaborative_same_second_title_does_not_collide(): second = Collaborative.objects.create(title=same_title, user=user) assert first.title == second.title == same_title + # The slug is also the subdomain, so it stays unique. + assert first.slug != second.slug assert first.public_id != second.public_id assert first.url_slug != second.url_slug From 8cc14530068d9161a759dce9acf1ff2b13bd6516 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Tue, 3 Nov 2026 17:28:20 +0530 Subject: [PATCH 41/55] chore(api): renumber public link migrations after dev's 0059 --- api/migrations/{0058_public_ids.py => 0060_public_ids.py} | 4 ++-- .../{0059_fill_public_ids.py => 0061_fill_public_ids.py} | 2 +- .../{0060_public_ids_unique.py => 0062_public_ids_unique.py} | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename api/migrations/{0058_public_ids.py => 0060_public_ids.py} (92%) rename api/migrations/{0059_fill_public_ids.py => 0061_fill_public_ids.py} (92%) rename api/migrations/{0060_public_ids_unique.py => 0062_public_ids_unique.py} (91%) diff --git a/api/migrations/0058_public_ids.py b/api/migrations/0060_public_ids.py similarity index 92% rename from api/migrations/0058_public_ids.py rename to api/migrations/0060_public_ids.py index b21e6cde..587bb998 100644 --- a/api/migrations/0058_public_ids.py +++ b/api/migrations/0060_public_ids.py @@ -9,11 +9,11 @@ class Migration(migrations.Migration): Also drops the unique rule on dataset, publication and use case slugs and on use case / collaborative titles: links use public_id, so titles may repeat. Collaborative slugs stay unique (the slug is also the subdomain). - Filled for existing rows in 0059, made unique and required in 0060. + Filled for existing rows in 0061, made unique and required in 0062. """ dependencies = [ - ("api", "0057_repair_event_search_index"), + ("api", "0059_event_search_cover_url"), ] operations = [ diff --git a/api/migrations/0059_fill_public_ids.py b/api/migrations/0061_fill_public_ids.py similarity index 92% rename from api/migrations/0059_fill_public_ids.py rename to api/migrations/0061_fill_public_ids.py index db54577a..2592705a 100644 --- a/api/migrations/0059_fill_public_ids.py +++ b/api/migrations/0061_fill_public_ids.py @@ -12,7 +12,7 @@ def fill(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ("api", "0058_public_ids"), + ("api", "0060_public_ids"), ] operations = [ diff --git a/api/migrations/0060_public_ids_unique.py b/api/migrations/0062_public_ids_unique.py similarity index 91% rename from api/migrations/0060_public_ids_unique.py rename to api/migrations/0062_public_ids_unique.py index f17c6350..137d124b 100644 --- a/api/migrations/0060_public_ids_unique.py +++ b/api/migrations/0062_public_ids_unique.py @@ -5,7 +5,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0059_fill_public_ids"), + ("api", "0061_fill_public_ids"), ] operations = [ From 44349731ccd97046533f5d98a149c5b6d1b6f910 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 4 Nov 2026 13:04:01 +0530 Subject: [PATCH 42/55] fix(api): stop same-name uploads failing on the resource slug rule Resource slugs were unique and the free one was picked before the insert, so two uploads of the same file name at the same moment could clash. Nothing looks a resource up by slug, so the unique rule is dropped. Fixes #229 --- .../0063_resource_slug_not_unique.py | 21 ++++++++++++++++++ api/models/Resource.py | 12 ++++------ tests/test_slug_collision.py | 22 ++++++++++++++++++- 3 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 api/migrations/0063_resource_slug_not_unique.py diff --git a/api/migrations/0063_resource_slug_not_unique.py b/api/migrations/0063_resource_slug_not_unique.py new file mode 100644 index 00000000..4b8169be --- /dev/null +++ b/api/migrations/0063_resource_slug_not_unique.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Resource slugs no longer need to be unique (#229). + + Nothing looks a resource up by slug, and the unique rule made two uploads + of the same file name at the same moment fail. + """ + + dependencies = [ + ("api", "0062_public_ids_unique"), + ] + + operations = [ + migrations.AlterField( + model_name="resource", + name="slug", + field=models.SlugField(max_length=255), + ), + ] diff --git a/api/models/Resource.py b/api/models/Resource.py index 9d20d934..2ed78501 100644 --- a/api/models/Resource.py +++ b/api/models/Resource.py @@ -41,7 +41,9 @@ class Resource(models.Model): ) name = models.CharField(max_length=200, unique=False, blank=False) description = models.TextField(blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Not unique: nothing looks a resource up by slug, and a unique slug made + # simultaneous uploads of the same file name fail (#229). + slug = models.SlugField(max_length=255) url = models.URLField(max_length=500) is_active = models.BooleanField(default=True) preview_enabled = models.BooleanField(default=False) @@ -53,13 +55,7 @@ class Resource(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - base_slug = slugify(self.name) - slug = base_slug - counter = 1 - while Resource.objects.filter(slug=slug).exclude(pk=self.pk).exists(): - slug = f"{base_slug}-{counter}" - counter += 1 - self.slug = slug + self.slug = slugify(self.name)[:255] super().save(*args, **kwargs) def __str__(self) -> str: diff --git a/tests/test_slug_collision.py b/tests/test_slug_collision.py index a4ae31ea..7cddb8d9 100644 --- a/tests/test_slug_collision.py +++ b/tests/test_slug_collision.py @@ -11,7 +11,7 @@ import pytest -from api.models import Collaborative, Dataset, UseCase +from api.models import Collaborative, Dataset, Resource, UseCase from authorization.models import User @@ -55,3 +55,23 @@ def test_collaborative_same_second_title_does_not_collide(): assert first.slug != second.slug assert first.public_id != second.public_id assert first.url_slug != second.url_slug + + +@pytest.mark.django_db +def test_resources_with_the_same_name_do_not_collide(): + # Resource slugs are no longer unique (#229): two uploads of the same file + # name used to race on the unique rule. + dataset = Dataset.objects.create(title="Resource collision") + first = Resource.objects.create(dataset=dataset, name="data.csv") + second = Resource.objects.create(dataset=dataset, name="data.csv") + + assert first.slug == second.slug == "datacsv" + assert first.pk != second.pk + + +@pytest.mark.django_db +def test_resource_with_a_preset_slug_keeps_it(): + dataset = Dataset.objects.create(title="Resource preset slug") + resource = Resource.objects.create(dataset=dataset, name="data.csv", slug="my-file") + + assert resource.slug == "my-file" From 4ddee7661aaa59d0b777ebd7441aaa75d47756e2 Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Wed, 4 Nov 2026 13:50:38 +0530 Subject: [PATCH 43/55] don't index summary for now --- api/views/search_usecase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/views/search_usecase.py b/api/views/search_usecase.py index a0edeaf7..60b04ab2 100644 --- a/api/views/search_usecase.py +++ b/api/views/search_usecase.py @@ -163,7 +163,6 @@ def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: searchable_fields = [ "title", - "summary", "tags", "sectors", "user.name", From 335d1c1782794660f25cf04abbdeaf58e67bfcaa Mon Sep 17 00:00:00 2001 From: Amit Chaudhary Date: Wed, 4 Nov 2026 14:45:16 +0530 Subject: [PATCH 44/55] fix(search): index long text fields without single-token sub-fields Summaries, AI model and publication descriptions, and user bios have no effective length limit, but their `raw` sub-fields used the keyword analyzer, which emits the whole value as one token. Lucene rejects tokens over 32,766 bytes, so Elasticsearch refused the whole document and the record never reached search. Nothing queries these sub-fields, so they are removed. The fields stay indexed and returned in results. This also makes the workaround in 4ddee76 unnecessary, so summary goes back into the use case search fields. Existing indexes keep the old mapping until rebuilt, and sync_search_mappings never removes fields. After deploying, each environment needs: python manage.py search_index --rebuild -f --models api.UseCase \ api.Collaborative api.AIModel api.Publication api.Organization \ authorization.User --- api/views/search_usecase.py | 1 + search/documents/aimodel_document.py | 8 +-- search/documents/collaborative_document.py | 8 +-- search/documents/publication_document.py | 6 +-- search/documents/publisher_document.py | 16 ++---- search/documents/usecase_document.py | 8 +-- tests/test_search_long_text_mapping.py | 61 ++++++++++++++++++++++ 7 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 tests/test_search_long_text_mapping.py diff --git a/api/views/search_usecase.py b/api/views/search_usecase.py index 9b271b42..a6e9a639 100644 --- a/api/views/search_usecase.py +++ b/api/views/search_usecase.py @@ -168,6 +168,7 @@ def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: searchable_fields = [ "title", + "summary", "tags", "sectors", "user.name", diff --git a/search/documents/aimodel_document.py b/search/documents/aimodel_document.py index 400da070..81c73c08 100644 --- a/search/documents/aimodel_document.py +++ b/search/documents/aimodel_document.py @@ -36,12 +36,8 @@ class AIModelDocument(Document): fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) version = fields.KeywordField() provider = fields.KeywordField() diff --git a/search/documents/collaborative_document.py b/search/documents/collaborative_document.py index 3ab5495f..52f09682 100644 --- a/search/documents/collaborative_document.py +++ b/search/documents/collaborative_document.py @@ -63,12 +63,8 @@ class CollaborativeDocument(Document): fields=title_subfields(), ) - summary = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit + summary = fields.TextField(analyzer=html_strip) logo = fields.TextField(analyzer=ngram_analyser) cover_image = fields.TextField(analyzer=ngram_analyser) diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py index e5fd45a7..f3c55558 100644 --- a/search/documents/publication_document.py +++ b/search/documents/publication_document.py @@ -35,10 +35,8 @@ class PublicationDocument(Document): analyzer=ngram_analyser, fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={"raw": fields.TextField(analyzer="keyword")}, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) status = fields.KeywordField() # Resource Type facet (name). diff --git a/search/documents/publisher_document.py b/search/documents/publisher_document.py index b63744d1..7273ad82 100644 --- a/search/documents/publisher_document.py +++ b/search/documents/publisher_document.py @@ -27,12 +27,8 @@ class PublisherDocument(Document): }, ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: user bios can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) publisher_type = fields.KeywordField() # 'organization' or 'user' @@ -47,12 +43,8 @@ class PublisherDocument(Document): location = fields.TextField(analyzer=ngram_analyser) # User specific fields - bio = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: bios can exceed Lucene's 32,766-byte token limit + bio = fields.TextField(analyzer=html_strip) profile_picture = fields.TextField(analyzer=ngram_analyser) username = fields.KeywordField() email = fields.KeywordField() diff --git a/search/documents/usecase_document.py b/search/documents/usecase_document.py index 9e292d65..f7872c90 100644 --- a/search/documents/usecase_document.py +++ b/search/documents/usecase_document.py @@ -47,12 +47,8 @@ class UseCaseDocument(Document): fields=title_subfields(), ) - summary = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: summaries can exceed Lucene's 32,766-byte token limit + summary = fields.TextField(analyzer=html_strip) logo = fields.TextField(analyzer=ngram_analyser) diff --git a/tests/test_search_long_text_mapping.py b/tests/test_search_long_text_mapping.py new file mode 100644 index 00000000..64ee7efe --- /dev/null +++ b/tests/test_search_long_text_mapping.py @@ -0,0 +1,61 @@ +"""Regression test for long text fields failing to index. + +Use case and collaborative summaries are rich-text JSON documents of up to +200,000 characters, and AI model descriptions, publication descriptions and user +bios have no length limit either. Their ``raw`` sub-fields used the ``keyword`` +analyzer, which emits the whole value as one token, and Lucene rejects any token +over 32,766 bytes. Elasticsearch then refused the whole document with an +"immense term" error, so the record never reached search. + +Nothing queries these sub-fields, so they are gone. The fields themselves stay +indexed: search results return them and text queries match on them. The publisher +``description`` is included because the user index fills it from ``bio``. +""" + +import pytest +from django.core.cache import cache + +from api.views.search_usecase import SearchUseCase +from search.documents.aimodel_document import AIModelDocument +from search.documents.collaborative_document import CollaborativeDocument +from search.documents.publication_document import PublicationDocument +from search.documents.publisher_document import ( + OrganizationPublisherDocument, + UserPublisherDocument, +) +from search.documents.usecase_document import UseCaseDocument + +LONG_TEXT_FIELDS = [ + (UseCaseDocument, "summary"), + (CollaborativeDocument, "summary"), + (AIModelDocument, "description"), + (PublicationDocument, "description"), + (OrganizationPublisherDocument, "description"), + (OrganizationPublisherDocument, "bio"), + (UserPublisherDocument, "description"), + (UserPublisherDocument, "bio"), +] + + +@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS) +def test_long_text_field_has_no_single_token_subfield(document, field): + mapping = document._doc_type.mapping.to_dict()["properties"][field] + + assert "raw" not in mapping.get("fields", {}) + + +@pytest.mark.parametrize("document,field", LONG_TEXT_FIELDS) +def test_long_text_field_stays_searchable(document, field): + mapping = document._doc_type.mapping.to_dict()["properties"][field] + + assert mapping["type"] == "text" + assert mapping["analyzer"] == "html_strip" + + +@pytest.mark.django_db +def test_usecase_search_matches_on_summary(): + cache.delete("usecase_search_metadata_config") + + searchable_fields, _ = SearchUseCase().get_searchable_and_aggregations() + + assert "summary" in searchable_fields From 657457feb6a7b354c50c8d410e3876f22e3c9c41 Mon Sep 17 00:00:00 2001 From: Anant Jain Date: Wed, 4 Nov 2026 23:58:56 +0530 Subject: [PATCH 45/55] fix(search): rebuild indexes that still have single-token long text copies Removing the keyword-analyzed raw sub-fields changed the documents only; existing indexes kept the old mapping, so a long use case summary still failed to index on save and every deploy failed at search_index --populate. Migration 0064 drops each search index whose live mapping still has such a sub-field and creates it again with the current mapping; the deploy then refills it. Also removes the same sub-field from dataset description. Fixes #266 --- .../0064_rebuild_long_text_search_indexes.py | 71 ++++++++++++++++++ search/documents/dataset_document.py | 8 +- tests/test_rebuild_long_text_indexes.py | 73 +++++++++++++++++++ tests/test_search_long_text_mapping.py | 2 + 4 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 api/migrations/0064_rebuild_long_text_search_indexes.py create mode 100644 tests/test_rebuild_long_text_indexes.py diff --git a/api/migrations/0064_rebuild_long_text_search_indexes.py b/api/migrations/0064_rebuild_long_text_search_indexes.py new file mode 100644 index 00000000..818498ae --- /dev/null +++ b/api/migrations/0064_rebuild_long_text_search_indexes.py @@ -0,0 +1,71 @@ +"""One-time rebuild of search indexes that still have a single-token text copy. + +Long text fields (use case and collaborative summaries, descriptions, bios) had +a ``raw`` sub-field with the ``keyword`` analyzer, which turns the whole value +into one token. Elasticsearch refuses tokens over 32,766 bytes, so a long +summary failed to index: saving a published item returned an error (#266) and +every deploy failed at ``search_index --populate``. The documents no longer +define those sub-fields, but an existing index keeps its old mapping. + +For each search index whose live mapping still has such a sub-field, drop it +and create it again empty with the current mapping. The deploy's release step +runs migrations first and then populates every index from the database, so +search is refilled right after. Nothing in the database changes. + +Does nothing when an index is missing or already clean, and never fails the +migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def _has_single_token_text(properties): + for field in properties.values(): + for sub in field.get("fields", {}).values(): + if sub.get("type") == "text" and sub.get("analyzer") == "keyword": + return True + if _has_single_token_text(field.get("properties", {})): + return True + return False + + +def rebuild_long_text_indexes(apps, schema_editor): + try: + from django_elasticsearch_dsl.registries import registry + from elasticsearch_dsl.connections import connections + + es = connections.get_connection().options(request_timeout=30) + indices = {index._name: index for index in registry.get_indices()} + for name in set(settings.ELASTICSEARCH_INDEX_NAMES.values()): + if not es.indices.exists(index=name): + continue + mappings = es.indices.get_mapping(index=name) + if not any( + _has_single_token_text(body.get("mappings", {}).get("properties", {})) + for body in mappings.values() + ): + continue + for concrete in mappings: + es.indices.delete(index=concrete) + # Create it again at once so a write from the running server cannot + # make Elasticsearch create it with guessed field types. + if name in indices: + indices[name].create() + logger.info("long_text_search_index_rebuilt", index=name) + except Exception as exc: # never block a deploy on search problems + logger.error("long_text_search_index_rebuild_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0063_resource_slug_not_unique"), + ] + + operations = [ + migrations.RunPython(rebuild_long_text_indexes, migrations.RunPython.noop), + ] diff --git a/search/documents/dataset_document.py b/search/documents/dataset_document.py index e144419e..bce09ecd 100644 --- a/search/documents/dataset_document.py +++ b/search/documents/dataset_document.py @@ -47,12 +47,8 @@ class DatasetDocument(Document): fields=title_subfields(), ) - description = fields.TextField( - analyzer=html_strip, - fields={ - "raw": fields.TextField(analyzer="keyword"), - }, - ) + # No keyword-analyzed sub-field: descriptions can exceed Lucene's 32,766-byte token limit + description = fields.TextField(analyzer=html_strip) status = fields.KeywordField() diff --git a/tests/test_rebuild_long_text_indexes.py b/tests/test_rebuild_long_text_indexes.py new file mode 100644 index 00000000..0143409b --- /dev/null +++ b/tests/test_rebuild_long_text_indexes.py @@ -0,0 +1,73 @@ +"""Migration 0064 rebuilds only search indexes that still have a single-token copy.""" + +import importlib +from unittest import mock + +from django.test import override_settings + +migration = importlib.import_module("api.migrations.0064_rebuild_long_text_search_indexes") + +OLD = { + "properties": { + "summary": { + "type": "text", + "fields": {"raw": {"type": "text", "analyzer": "keyword"}}, + } + } +} +CLEAN = { + "properties": { + "summary": {"type": "text", "analyzer": "html_strip"}, + "title": {"type": "text", "fields": {"raw": {"type": "keyword"}}}, + } +} +NAMES = {"a": "usecase", "b": "dataset", "c": "event"} + + +def _run(mappings): + es = mock.MagicMock() + es.options.return_value = es + es.indices.exists.side_effect = lambda index: index in mappings + es.indices.get_mapping.side_effect = lambda index: {index: {"mappings": mappings[index]}} + indices = {name: mock.MagicMock(_name=name) for name in NAMES.values()} + with ( + override_settings(ELASTICSEARCH_INDEX_NAMES=NAMES), + mock.patch("elasticsearch_dsl.connections.connections.get_connection", return_value=es), + mock.patch( + "django_elasticsearch_dsl.registries.registry.get_indices", + return_value=list(indices.values()), + ), + ): + migration.rebuild_long_text_indexes(None, None) + return es, indices + + +def test_old_index_is_dropped_and_created_again(): + es, indices = _run({"usecase": OLD, "dataset": CLEAN}) + + es.indices.delete.assert_called_once_with(index="usecase") + indices["usecase"].create.assert_called_once() + indices["dataset"].create.assert_not_called() + + +def test_nested_single_token_copy_is_found(): + nested = {"properties": {"datasets": {"type": "nested", "properties": OLD["properties"]}}} + + es, _ = _run({"dataset": nested}) + + es.indices.delete.assert_called_once_with(index="dataset") + + +def test_clean_or_missing_indexes_are_left_alone(): + es, indices = _run({"usecase": CLEAN}) + + es.indices.delete.assert_not_called() + assert not any(index.create.called for index in indices.values()) + + +def test_search_errors_never_fail_the_migration(): + with mock.patch( + "elasticsearch_dsl.connections.connections.get_connection", + side_effect=ConnectionError("down"), + ): + migration.rebuild_long_text_indexes(None, None) diff --git a/tests/test_search_long_text_mapping.py b/tests/test_search_long_text_mapping.py index 64ee7efe..28d52f9d 100644 --- a/tests/test_search_long_text_mapping.py +++ b/tests/test_search_long_text_mapping.py @@ -18,6 +18,7 @@ from api.views.search_usecase import SearchUseCase from search.documents.aimodel_document import AIModelDocument from search.documents.collaborative_document import CollaborativeDocument +from search.documents.dataset_document import DatasetDocument from search.documents.publication_document import PublicationDocument from search.documents.publisher_document import ( OrganizationPublisherDocument, @@ -28,6 +29,7 @@ LONG_TEXT_FIELDS = [ (UseCaseDocument, "summary"), (CollaborativeDocument, "summary"), + (DatasetDocument, "description"), (AIModelDocument, "description"), (PublicationDocument, "description"), (OrganizationPublisherDocument, "description"), From c634d13041f8d420638a07e78c1465155f31fa00 Mon Sep 17 00:00:00 2001 From: psaesha Date: Fri, 13 Nov 2026 14:59:49 +0530 Subject: [PATCH 46/55] fix(data-api): read tables whose column names contain a percent sign A table with a header like "Prevalence of Anaemia (%)" could not be read through the dataset/resource data endpoints: every request that selected, filtered or sorted on that column failed with "Failed to fetch data" (500). psycopg2 fills in %s placeholders by %-formatting the whole query text, including quoted column names, so the lone % broke the query. Column names are now quoted through one helper that doubles the %, which the driver turns back into a single % when it fills in the parameters. Checked against a real Postgres for plain reads, equality, contains + sort, and is-null filters. Tests: tests/test_data_indexing_filters.py::TestColumnNamesWithPercentSign Co-Authored-By: Claude Opus 5.5 --- api/utils/data_indexing.py | 25 ++++++++++++++++++++----- tests/test_data_indexing_filters.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 16201045..9ac20feb 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -532,6 +532,18 @@ def get_resource_columns(resource: Resource) -> List[str]: return [] +def _column_ident(column: str) -> pg_sql.Identifier: + """Quote a column name for a query that will be run with parameters. + + psycopg2 fills in ``%s`` placeholders by %-formatting the whole query + text, quoted names included, so a header like "Literacy rate (%)" breaks + the query unless its ``%`` is doubled. Every query in + ``fetch_resource_data`` is run with a parameter list, so the doubled + ``%%`` always comes back out as ``%``. + """ + return pg_sql.Identifier(column.replace("%", "%%")) + + def _parse_filter_key(key: str) -> Tuple[str, str]: """Split 'col__op' style filter key into (column, op). Defaults op to 'eq'.""" if "__" in key: @@ -564,7 +576,7 @@ def _build_where_clause( raise DataFetchError(f"Unknown filter column: {col}") op_template, transformer = _FILTER_OPERATORS[op] - col_ident = pg_sql.Identifier(col) + col_ident = _column_ident(col) if op in ("isnull", "notnull"): # Boolean toggle: isnull=true means IS NULL, isnull=false means IS NOT NULL truthy = value not in (False, "false", "False", 0, "0", None) @@ -596,12 +608,15 @@ def _build_order_by(order_by: Optional[List[str]], allowed_columns: List[str]) - col = item[1:] if col not in allowed_set: raise DataFetchError(f"Unknown order_by column: {col}") - parts.append( - pg_sql.SQL("{col} ").format(col=pg_sql.Identifier(col)) + pg_sql.SQL(direction) - ) + parts.append(pg_sql.SQL("{col} ").format(col=_column_ident(col)) + pg_sql.SQL(direction)) return pg_sql.SQL(" ORDER BY ") + pg_sql.SQL(", ").join(parts) +def _build_select_list(columns: List[str]) -> pg_sql.Composable: + """Build the comma-separated list of quoted column names to SELECT.""" + return pg_sql.SQL(", ").join(_column_ident(c) for c in columns) + + def fetch_resource_data( resource: Resource, filters: Optional[Dict[str, Any]] = None, @@ -661,7 +676,7 @@ def fetch_resource_data( offset = max(int(offset or 0), 0) table_ident = pg_sql.Identifier(data_table.table_name) - cols_sql = pg_sql.SQL(", ").join(pg_sql.Identifier(c) for c in select_columns) + cols_sql = _build_select_list(select_columns) where_sql, params = _build_where_clause(filters or {}, allowed_columns) order_sql = _build_order_by(order_by, allowed_columns) diff --git a/tests/test_data_indexing_filters.py b/tests/test_data_indexing_filters.py index e8db64ff..06d71dd9 100644 --- a/tests/test_data_indexing_filters.py +++ b/tests/test_data_indexing_filters.py @@ -15,6 +15,7 @@ from api.utils.data_indexing import ( DataFetchError, _build_order_by, + _build_select_list, _build_where_clause, _parse_filter_key, ) @@ -124,6 +125,34 @@ def test_unknown_column_rejected(self) -> None: _build_order_by(["evil"], self.allowed) +def _fill_params(query: str, params: list) -> str: + """Fill in ``%s`` placeholders the way psycopg2 does before sending a query. + + psycopg2 runs the whole query text through %-formatting whenever a + parameter list is passed (even an empty one), so a lone ``%`` anywhere in + the text, including inside a quoted column name, breaks it. + """ + return query % tuple(f"'{p}'" for p in params) + + +class TestColumnNamesWithPercentSign(unittest.TestCase): + """Indicator tables often have headers like "Literacy rate (%)".""" + + column = "Prevalence of Anaemia Among Women of Age 15 to 49 (%)" + + def test_select_list_keeps_the_column_name_after_params_are_filled(self) -> None: + sql = _build_select_list([self.column, "District"]) + self.assertEqual(_fill_params(_render(sql), []), f'"{self.column}", "District"') + + def test_filter_keeps_the_column_name_after_params_are_filled(self) -> None: + sql, params = _build_where_clause({self.column: "Kamrup"}, [self.column]) + self.assertEqual(_fill_params(_render(sql), params), f" WHERE \"{self.column}\" = 'Kamrup'") + + def test_order_by_keeps_the_column_name_after_params_are_filled(self) -> None: + sql = _build_order_by([f"-{self.column}"], [self.column]) + self.assertEqual(_fill_params(_render(sql), []), f' ORDER BY "{self.column}" DESC') + + class TestViewQueryParamHelpers(unittest.TestCase): def test_parse_bool(self) -> None: self.assertTrue(_parse_bool("true")) From f4b743d54246deb16097590cfb406e8aca8c2825 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 13 Nov 2026 17:36:27 +0530 Subject: [PATCH 47/55] ci: pass test accounts 3-5 to the CivicDataSpace-test smoke workflow --- .github/workflows/deploy-backend.yml | 12 ++++++++++++ .github/workflows/deploy-to-ecs.yml | 6 ++++++ .github/workflows/pr-gate.yml | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index b723adeb..efb1001b 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -288,6 +288,12 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a # confidential client, so without this the token request returns 401 # and the job fails its preflight. @@ -310,6 +316,12 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} tests-skipped: diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index f3573b94..67603e97 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -158,6 +158,12 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a # confidential client, so without this the token request returns 401 # and the job fails its preflight. diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 5a9f6ce3..98829b06 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -48,4 +48,10 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} From 87baabacbdbcd7857893970bb8044c3a1e500040 Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 01:03:47 +0530 Subject: [PATCH 48/55] feat(data-index): give every upload its own table and switch over safely Table names were built from the resource name and dataset title, and Postgres keeps only the first 63 bytes of a name, so two resources with long similar names shared one table and a re-upload of one replaced the other's data. Re-uploads also dropped the old table before the new one existed, so readers saw empty or half-written data and a failed re-upload lost it. Each upload now gets a table named rdt_. index_resource_data builds the new table in one data-database transaction, then switches the resource over in one main-database commit (resource row locked, stored file re-checked so the newest file wins, column rows rebuilt with errors escaping) and confirms the new row exists; anything short of that drops this build's table and keeps the old data. A post_delete receiver on ResourceDataTable drops the old table after commit, which also frees storage on resource, dataset and admin deletes; it leaves legacy tables two rows still share. update_dataindex no longer wraps each call in a transaction, and the flow refuses to run inside one. Tests run against a real Postgres when TEST_DATA_DB_URL is set; CI adds a postgres service and fails instead of skipping them. Plan: plans/claude/data-index-table-names.md (Phase 1) --- .github/workflows/django-tests.yml | 18 + api/management/commands/update_dataindex.py | 7 +- api/models/Resource.py | 8 +- api/signals/__init__.py | 1 + api/signals/data_table_signals.py | 33 ++ api/utils/data_indexing.py | 405 +++++------------ api/utils/data_table_names.py | 44 ++ api/utils/data_tables.py | 378 ++++++++++++++++ pytest.ini | 4 +- tests/data_index/__init__.py | 0 tests/data_index/conftest.py | 111 +++++ tests/data_index/fixtures/districts_v1.csv | 4 + tests/data_index/fixtures/districts_v2.csv | 5 + tests/data_index/fixtures/long_header.csv | 3 + tests/data_index/fixtures/out_of_range.csv | 3 + tests/data_index/fixtures/percent_header.csv | 4 + tests/data_index/routers.py | 18 + tests/data_index/support.py | 60 +++ tests/data_index/test_delete_cleanup.py | 181 ++++++++ tests/data_index/test_reader_regression.py | 59 +++ tests/data_index/test_reindex_switch.py | 434 +++++++++++++++++++ tests/data_index/test_table_names.py | 93 ++++ tests/data_index/test_upload_mutations.py | 128 ++++++ tests/test_settings.py | 12 + 24 files changed, 1697 insertions(+), 316 deletions(-) create mode 100644 api/signals/data_table_signals.py create mode 100644 api/utils/data_table_names.py create mode 100644 api/utils/data_tables.py create mode 100644 tests/data_index/__init__.py create mode 100644 tests/data_index/conftest.py create mode 100644 tests/data_index/fixtures/districts_v1.csv create mode 100644 tests/data_index/fixtures/districts_v2.csv create mode 100644 tests/data_index/fixtures/long_header.csv create mode 100644 tests/data_index/fixtures/out_of_range.csv create mode 100644 tests/data_index/fixtures/percent_header.csv create mode 100644 tests/data_index/routers.py create mode 100644 tests/data_index/support.py create mode 100644 tests/data_index/test_delete_cleanup.py create mode 100644 tests/data_index/test_reader_regression.py create mode 100644 tests/data_index/test_reindex_switch.py create mode 100644 tests/data_index/test_table_names.py create mode 100644 tests/data_index/test_upload_mutations.py diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 6bca8c20..2f49c740 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -16,6 +16,20 @@ jobs: django-tests: runs-on: ubuntu-latest + # The data database: tests marked data_db build real tables in it. + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: - uses: actions/checkout@v4 @@ -40,6 +54,10 @@ jobs: # tests/object_types/charts and the dataset metadata tests were left # behind by the chart consolidation and fail on main: see #189. - name: Run tests + env: + TEST_DATA_DB_URL: postgres://postgres:postgres@localhost:5432/postgres + # Fail the run instead of skipping the data_db tests if the database is missing. + REQUIRE_DATA_DB_TESTS: "1" run: | pytest tests/ \ --ignore=tests/object_types/charts \ diff --git a/api/management/commands/update_dataindex.py b/api/management/commands/update_dataindex.py index af1190a4..b9647f77 100644 --- a/api/management/commands/update_dataindex.py +++ b/api/management/commands/update_dataindex.py @@ -4,7 +4,6 @@ import structlog from django.core.management.base import BaseCommand, CommandParser -from django.db import transaction from api.models.Resource import Resource, ResourceDataTable from api.utils.data_indexing import INDEXED_FORMATS, index_resource_data @@ -125,9 +124,9 @@ def handle(self, *args: Any, **options: Dict[str, Any]) -> None: f"[{i}/{total_resources}] Processing resource {resource.id} - {resource.name}" ) - # Update dataindex for the resource - with transaction.atomic(): - data_table = index_resource_data(resource) + # Not inside a transaction: the flow commits its own switch and refuses + # to run inside an outer one (a failed resource keeps its old data). + data_table = index_resource_data(resource) if data_table: success_count += 1 diff --git a/api/models/Resource.py b/api/models/Resource.py index 2ed78501..f3f54d3a 100644 --- a/api/models/Resource.py +++ b/api/models/Resource.py @@ -11,6 +11,7 @@ logger = structlog.getLogger(__name__) +from api.utils.data_table_names import data_table_name from api.utils.enums import DataType from api.utils.storage import private_storage @@ -101,11 +102,8 @@ def __str__(self): def save(self, *args, **kwargs): if not self.table_name: - # Generate a unique table name based on resource ID - safe_name = slugify(self.resource.name)[:50] - safe_title = slugify(self.resource.dataset.title)[:50] - resource_id = self.resource.id.hex[:8] # Use shorter ID - self.table_name = f"{safe_name}_{safe_title}_{resource_id}" + # Named from this row's own id: unique, and never cut by Postgres's 63-byte limit. + self.table_name = data_table_name(self.id) super().save(*args, **kwargs) diff --git a/api/signals/__init__.py b/api/signals/__init__.py index 3d504230..141dbf97 100644 --- a/api/signals/__init__.py +++ b/api/signals/__init__.py @@ -1,6 +1,7 @@ # Import signals to register them from api.signals import ( aimodel_signals, + data_table_signals, dataset_signals, event_signals, publication_signals, diff --git a/api/signals/data_table_signals.py b/api/signals/data_table_signals.py new file mode 100644 index 00000000..655ebec0 --- /dev/null +++ b/api/signals/data_table_signals.py @@ -0,0 +1,33 @@ +"""Drop a data table once the row that points at it is deleted. + +Re-uploads, resource deletes, dataset deletes (by cascade) and admin deletes +all delete the resource's ResourceDataTable row, so this one receiver frees the +storage for every one of them. The drop waits for the delete to commit, so a +rolled-back delete keeps its data, and it leaves a legacy table alone while +another row still shares it. + +Registered in api/signals/__init__.py (a test proves it is connected once the +app loads). Docs: ../utils/data_indexing_architecture.md +""" + +from functools import partial +from typing import Any + +from django.db import transaction +from django.db.models.signals import post_delete +from django.dispatch import receiver + +from api.models.Resource import ResourceDataTable +from api.utils.data_tables import drop_data_table_if_unshared + + +@receiver(post_delete, sender=ResourceDataTable) +def drop_table_after_row_delete( + sender: Any, instance: ResourceDataTable, using: str, **kwargs: Any +) -> None: + # Once the delete commits, drop the row's table (unless another row still uses it). + transaction.on_commit( + partial(drop_data_table_if_unshared, instance.table_name, instance.pk), + using=using, + robust=True, + ) diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 9ac20feb..63080b41 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -1,23 +1,49 @@ -from typing import Any, Dict, Generator, List, Optional, Tuple +"""Copying tabular uploads into the data database, and reading them back. + +The upload flow is ``index_resource_data`` below; its helpers live in +./data_tables.py. The rest of this module reads the tables (row count, +preview, column list, the data API's filtered fetch, chart queries); every +reader looks the table up from the resource's ResourceDataTable row. + +Docs: ./data_indexing_architecture.md +""" + +from typing import Any, Dict, List, Optional, Tuple import pandas as pd import structlog from django.db import connections, transaction from django.db.utils import ProgrammingError +from django.utils import timezone from psycopg2 import sql as pg_sql # type: ignore[import-untyped] from api.models.Resource import Resource, ResourceDataTable from api.models.ResourceSchema import ResourceSchema from api.types.type_preview_data import PreviewData -from api.utils.file_utils import load_tabular_data, local_copy +from api.utils.data_tables import ( # noqa: F401 — DATA_DB, INDEXED_FORMATS, get_sql_type re-exported + DATA_DB, + INDEXED_FORMATS, + NewerFileSaved, + build_data_table, + data_table_row_exists, + describe_data_table, + drop_data_table, + error_fields, + get_sql_type, + has_indexable_file, + is_inside_transaction, + load_resource_frame, + lock_resource, + new_data_table_row, + rebuild_resource_schema, + retire_data_table_row, + save_data_table_row, + stored_file_identity, +) +from api.utils.file_utils import local_copy logger = structlog.get_logger("dataspace.data_indexing") -# Use a separate database for data tables -DATA_DB = "data_db" # This should match the connection name in settings.py - -# Formats indexed into ResourceDataTable (queryable via get_row_count) -INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} # Formats counted by parsing the file directly (not indexed into DB) FILE_COUNT_FORMATS = {"yml", "yaml", "xml"} @@ -43,313 +69,80 @@ } -def get_sql_type(pandas_dtype: str) -> str: - """Convert pandas dtype to SQL type.""" - if "int" in pandas_dtype: - return "BIGINT" - elif "float" in pandas_dtype: - return "DOUBLE PRECISION" - elif "datetime" in pandas_dtype: - return "TIMESTAMP" - elif "bool" in pandas_dtype: - return "BOOLEAN" - else: - return "TEXT" - - -def create_table_for_resource(resource: Resource, df: pd.DataFrame) -> Optional[ResourceDataTable]: - """Create a database table for the resource data and index it.""" - try: - # Create ResourceDataTable entry first to get the table name - data_table = ResourceDataTable.objects.create(resource=resource) - table_name = data_table.table_name - - # Get column types - column_types = {col: get_sql_type(str(df[col].dtype)) for col in df.columns} - - # Create table in the data database - with connections[DATA_DB].cursor() as cursor: - # Drop table if exists - cursor.execute(f'DROP TABLE IF EXISTS "{table_name}"') - - # Create table with proper column types - columns = [f'"{col}" {dtype}' for col, dtype in column_types.items()] - create_table_sql = f'CREATE TABLE "{table_name}" ({", ".join(columns)})' - cursor.execute(create_table_sql) - - # Create temporary table for COPY - temp_table = f"temp_{table_name}" - cursor.execute(f'CREATE TEMP TABLE "{temp_table}" (LIKE "{table_name}")') - - # Copy data to temp table - quoted_columns = [f'"{col}"' for col in df.columns] - # Use StringIO to create a file-like object for the CSV data - from io import StringIO - - csv_data = StringIO() - df.to_csv(csv_data, index=False, header=False) - csv_data.seek(0) - - copy_sql = f'COPY "{temp_table}" ({",".join(quoted_columns)}) FROM STDIN WITH CSV' - cursor.copy_expert(copy_sql, csv_data) - - # Insert from temp to main table with validation - cursor.execute(f'INSERT INTO "{table_name}" SELECT * FROM "{temp_table}"') - cursor.execute(f'DROP TABLE "{temp_table}"') - - return data_table - - except Exception as e: - logger.error(f"Error creating table for resource {resource.id}: {str(e)}") - if "data_table" in locals(): - data_table.delete() - return None - - def index_resource_data(resource: Resource) -> Optional[ResourceDataTable]: - """Index a resource's CSV data into a database table. - - This function handles the indexing process gracefully, catching and logging - specific errors at each stage of the process without crashing. - - Args: - resource: The Resource object to index - - Returns: - Optional[ResourceDataTable]: The created data table or None if indexing failed + """Copy a resource's tabular file into a table of its own in the data database. + + Every upload gets a new table, named from its new ResourceDataTable row. + The table is built in full first; then one main-database commit points the + resource at it and rebuilds the column list. The old table is dropped after + that commit by the row's delete receiver. Readers see the complete old data + until the commit, then the complete new data. If anything fails, the old + data stays as it was and this build's table is dropped. + + Must be called outside any transaction, so "committed" really means + committed. Returns the new ResourceDataTable, or None when the resource + was not switched over. """ - resource_id = getattr(resource, "id", "unknown") + resource_id = str(resource.id) - try: - # Check if resource is a supported tabular file - try: - file_details = resource.resourcefiledetails - if not file_details: - logger.info(f"Resource {resource_id} has no file details, skipping indexing") - return None - except Exception as e: - logger.error(f"Failed to access file details for resource {resource_id}: {str(e)}") - return None - - # Check file format - try: - format = file_details.format.lower() - supported_formats = [ - "csv", - "xls", - "xlsx", - "ods", - "parquet", - "feather", - "json", - "tsv", - ] - if format not in supported_formats: - logger.info( - f"Resource {resource_id} has unsupported format: {format}, skipping indexing" - ) - return None - except Exception as e: - logger.error(f"Failed to determine format for resource {resource_id}: {str(e)}") - return None - - # Load tabular data with timeout protection - try: - import signal - from contextlib import contextmanager - - @contextmanager - def timeout(seconds: int) -> Generator[None, None, None]: - def handler(signum: int, frame: Any) -> None: - raise TimeoutError(f"Loading data timed out after {seconds} seconds") - - # Set the timeout handler - original_handler = signal.getsignal(signal.SIGALRM) - signal.signal(signal.SIGALRM, handler) - signal.alarm(seconds) - try: - yield - finally: - signal.alarm(0) - signal.signal(signal.SIGALRM, original_handler) - - with local_copy(file_details.file) as file_path: - # Apply timeout only on Unix-based systems where signal.SIGALRM is available - try: - with timeout(60): # 60 second timeout for loading data - df = load_tabular_data(file_path, format) - except TimeoutError as te: - logger.error( - f"Timeout while loading data for resource {resource_id}: {str(te)}" - ) - return None - except Exception: - # Fallback without timeout if signal.SIGALRM is not available (e.g., on Windows) - df = load_tabular_data(file_path, format) - - if df is None: - logger.error( - f"Failed to load data for resource {resource_id}: Data loading returned None" - ) - return None - if df.empty: - logger.info(f"Resource {resource_id} has empty data, skipping indexing") - return None - - # Check for problematic column names and sanitize them - try: - # Replace problematic characters in column names - # Create a new list of sanitized column names - sanitized_columns = [ - str(col).replace('"', "").replace("\n", " ").replace("\t", " ") - for col in df.columns - ] - # Assign to df.columns using pandas Index constructor - df.columns = pd.Index(sanitized_columns) - - # Check for duplicate column names after sanitization - if len(df.columns) != len(set(df.columns)): - # Handle duplicate columns by adding suffixes - from collections import Counter - - col_counter = Counter(df.columns) - for col, count in col_counter.items(): - if count > 1: - # Find all occurrences of this column name - indices = [i for i, x in enumerate(df.columns) if x == col] - # Rename all but the first occurrence - for i, idx in enumerate(indices[1:], 1): - df.columns.values[idx] = f"{col}_{i}" - logger.warning(f"Renamed duplicate columns in resource {resource_id}") - except Exception as e: - logger.error( - f"Failed to sanitize column names for resource {resource_id}: {str(e)}" - ) - return None - - except Exception as e: - import traceback - - logger.error( - f"Failed to load data for resource {resource_id}: {str(e)}\n{traceback.format_exc()}" - ) - return None - - # Create and index table within transaction - try: - with transaction.atomic(): - # Delete existing table if any - try: - existing_table = ResourceDataTable.objects.get(resource=resource) - try: - with connections[DATA_DB].cursor() as cursor: - cursor.execute(f'DROP TABLE IF EXISTS "{existing_table.table_name}"') - except Exception as drop_error: - logger.error( - f"Failed to drop existing table for resource {resource_id}: {str(drop_error)}" - ) - # Continue anyway as we'll try to recreate the table - - existing_table.delete() - except ResourceDataTable.DoesNotExist: - pass - except Exception as e: - logger.error( - f"Error handling existing table for resource {resource_id}: {str(e)}" - ) - # Continue with table creation - - # Create new table - data_table = create_table_for_resource(resource, df) - if not data_table: - logger.error(f"Failed to create table for resource {resource_id}") - return None - - # Update resource schema - try: - # Store existing schema descriptions to preserve them during re-indexing - existing_schemas: Dict[str, Dict[str, Optional[str]]] = {} - for schema in ResourceSchema.objects.filter( - resource=resource - ): # type: ResourceSchema - existing_schemas[schema.field_name] = { - "description": schema.description, - "format": schema.format, - } - - # Clear existing schema for this resource - ResourceSchema.objects.filter(resource=resource).delete() - - # Then create new schema entries for each column, preserving descriptions where possible - schema_success_count = 0 - schema_error_count = 0 - - for col in df.columns: - try: - # Determine format - always use the detected format from the data - sql_type = get_sql_type(str(df[col].dtype)) - - # Map SQL types to FieldTypes enum values - if sql_type == "BIGINT": - format_value = "INTEGER" - elif sql_type == "DOUBLE PRECISION": - format_value = "NUMBER" - elif sql_type == "TIMESTAMP": - format_value = "DATE" - elif sql_type == "BOOLEAN": - format_value = "BOOLEAN" - else: # TEXT or any other type - format_value = "STRING" - - # For description, preserve existing if available, otherwise auto-generate - description = f"Description of column {col}" - if col in existing_schemas: - existing_description = existing_schemas[col]["description"] - # Check for None and non-auto-generated descriptions - if existing_description is not None: - description = existing_description - logger.debug(f"Preserved custom description for column {col}") - - # Create the schema entry - ResourceSchema.objects.create( - resource=resource, - field_name=col, - format=format_value, - description=description, - ) - schema_success_count += 1 - except Exception as schema_error: - schema_error_count += 1 - logger.error( - f"Error creating schema for column {col} in resource {resource_id}: {str(schema_error)}" - ) - # Continue with other columns even if one fails - - logger.info( - f"Resource {resource_id} schema creation: {schema_success_count} columns succeeded, {schema_error_count} failed" - ) - except Exception as schema_error: - logger.error( - f"Failed to update schema for resource {resource_id}: {str(schema_error)}" - ) - # Continue and return the data_table even if schema update fails - - return data_table - except Exception as tx_error: - import traceback + # Refuse to run inside someone else's transaction: the switch would only release a savepoint. + if is_inside_transaction(): + logger.error("data_index_refused_inside_transaction", resource_id=resource_id) + return None - logger.error( - f"Transaction error for resource {resource_id}: {str(tx_error)}\n{traceback.format_exc()}" - ) - return None + # Only supported tabular files (csv, xlsx, parquet, ...) get a table. + if not has_indexable_file(resource): + return None - except Exception as e: - import traceback + # Read the file into a frame, remembering exactly which stored file it was. + loaded = load_resource_frame(resource) + if loaded is None: + return None - logger.error( - f"Unexpected error indexing resource {resource_id}: {str(e)}\n{traceback.format_exc()}" + # This upload's own row, and with it its own table name. + new_row = new_data_table_row(resource) + switched = False + try: + # Build the complete new table; readers keep using the old one meanwhile. + build_data_table( + new_row.table_name, loaded.frame, describe_data_table(resource, built_at=timezone.now()) ) + + # Switch the resource over to the new table in one commit. + with transaction.atomic(): + # Hold the resource, so two uploads of it switch one after the other. + lock_resource(resource) + + # If a newer file was saved while we built, that upload wins: back off. + if stored_file_identity(resource) != loaded.file: + raise NewerFileSaved() + + # Retire the old row; its table is dropped once this commits. + previous_created = retire_data_table_row(resource) + + # Point the resource at the new table, keeping its place as the dataset's default. + save_data_table_row(new_row, created=previous_created) + + # Rebuild the column list, keeping the descriptions people wrote. + rebuild_resource_schema(resource, loaded.frame) + + # It only counts as switched if the new row really is there after the commit. + switched = data_table_row_exists(new_row.pk) + if not switched: + logger.error("data_index_switch_not_saved", resource_id=resource_id) + except NewerFileSaved: + logger.info("data_index_skipped_newer_file", resource_id=resource_id) + except Exception as error: + logger.error("data_index_failed", resource_id=resource_id, **error_fields(error)) + finally: + # Anything short of a confirmed switch leaves this build's table unused: drop it. + if not switched: + drop_data_table(new_row.table_name) + + if not switched: return None + logger.info("data_index_switched", resource_id=resource_id, table_name=new_row.table_name) + return new_row def query_resource_data(resource: Resource, query: str) -> Optional[pd.DataFrame]: diff --git a/api/utils/data_table_names.py b/api/utils/data_table_names.py new file mode 100644 index 00000000..7be81964 --- /dev/null +++ b/api/utils/data_table_names.py @@ -0,0 +1,44 @@ +"""How tables in the data database are named. + +Every tabular upload gets its own table, named "rdt_" plus the 32-character +hex of its ResourceDataTable row id (36 characters in all). The name never +depends on a resource or dataset title, so renames don't touch it and two +uploads can never share one. + +Postgres keeps only the first 63 bytes of any name. Names from before this +rule were built from titles (up to 110 characters), so two of them can map to +the same physical table; ``physical_table_name`` gives the name Postgres +actually stored, for the checks that care. + +No imports from models: the ResourceDataTable model uses this module. +Docs: ./data_indexing_architecture.md +""" + +import uuid + +DATA_TABLE_PREFIX = "rdt_" +MAX_IDENTIFIER_BYTES = 63 +# A cut name is at least this long: a cut never drops more than a 4-byte character's first 3 bytes. +SHORTEST_CUT_NAME_BYTES = MAX_IDENTIFIER_BYTES - 3 + + +def data_table_name(row_id: uuid.UUID) -> str: + """The table name for a ResourceDataTable row: "rdt_" + its id in lowercase hex.""" + return f"{DATA_TABLE_PREFIX}{row_id.hex}" + + +def physical_table_name(name: str) -> str: + """The name Postgres stores for ``name``: its first 63 bytes, never half a character.""" + encoded = name.encode("utf-8") + if len(encoded) <= MAX_IDENTIFIER_BYTES: + return name + return encoded[:MAX_IDENTIFIER_BYTES].decode("utf-8", errors="ignore") + + +def may_share_physical_table(name: str) -> bool: + """Whether another, longer name could be cut to the same physical table as this one. + + Only names at least as long as the shortest possible cut can; "rdt_" names + (36 bytes) never can. + """ + return len(name.encode("utf-8")) >= SHORTEST_CUT_NAME_BYTES diff --git a/api/utils/data_tables.py b/api/utils/data_tables.py new file mode 100644 index 00000000..de4603bc --- /dev/null +++ b/api/utils/data_tables.py @@ -0,0 +1,378 @@ +"""Helpers for copying uploaded files into tables in the data database. + +The upload flow (``index_resource_data`` in ./data_indexing.py) reads a +resource's file, builds a brand-new table for it, then switches the resource's +ResourceDataTable row over to that table in one main-database commit. Old tables +are dropped after the commit by the row's delete receiver +(api/signals/data_table_signals.py). Each helper here does one step of that. + +Tables live in the separate ``data_db`` database (its statements autocommit); +the rows that point at them (ResourceDataTable, ResourceSchema) live in the +main database. New SQL composes names and comments with psycopg2's +``sql.Identifier`` / ``sql.Literal``, never with string formatting. + +Logs carry ids, counts and error codes only — never file contents or values. +Docs: ./data_indexing_architecture.md +""" + +import json +import os +import signal +import threading +import traceback +from collections import Counter +from datetime import datetime +from io import StringIO +from typing import Any, Dict, List, NamedTuple, Optional + +import pandas as pd +import structlog +from django.db import connections, transaction +from psycopg2 import sql as pg_sql # type: ignore[import-untyped] + +from api.models.Resource import Resource, ResourceDataTable, ResourceFileDetails +from api.models.ResourceSchema import ResourceSchema +from api.utils.data_table_names import ( + data_table_name, + may_share_physical_table, + physical_table_name, +) +from api.utils.file_utils import load_tabular_data, local_copy + +logger = structlog.get_logger("dataspace.data_tables") + +DATA_DB = "data_db" # the connection name in settings.DATABASES + +# Formats copied into a table (and so readable through the data API). +INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} + +# How long a drop waits for readers holding the table before giving up. +DROP_LOCK_TIMEOUT = "5s" + +# How long reading a file may take, where the platform lets us stop it. +LOAD_TIMEOUT_SECONDS = 60 + + +class FileIdentity(NamedTuple): + """Which stored file a build read: its storage name and SHA-256.""" + + name: str + sha256: Optional[str] + + +class LoadedFile(NamedTuple): + """A resource's file read into a frame, with the identity of that file.""" + + frame: pd.DataFrame + file: FileIdentity + + +class NewerFileSaved(Exception): + """Another upload saved a newer file while this one was being built.""" + + +def get_sql_type(pandas_dtype: str) -> str: + """The Postgres column type for a pandas dtype name (TEXT when unsure).""" + if "int" in pandas_dtype: + return "BIGINT" + if "float" in pandas_dtype: + return "DOUBLE PRECISION" + if "datetime" in pandas_dtype: + return "TIMESTAMP" + if "bool" in pandas_dtype: + return "BOOLEAN" + return "TEXT" + + +_FIELD_FORMATS = { + "BIGINT": "INTEGER", + "DOUBLE PRECISION": "NUMBER", + "TIMESTAMP": "DATE", + "BOOLEAN": "BOOLEAN", +} + + +def error_fields(error: BaseException) -> Dict[str, Any]: + """Log fields for an error: its class, Postgres error code and where it was raised. + + Never its message: database messages quote the offending value (a COPY + error names the cell). ``where`` is the innermost frame in our own code + ("data_tables.py:231 build_data_table"), which carries no data. + """ + cause = error.__cause__ or error + frames = traceback.extract_tb(error.__traceback__) + ours = [f for f in frames if f"{os.sep}api{os.sep}" in f.filename] or frames + where = ( + f"{os.path.basename(ours[-1].filename)}:{ours[-1].lineno} {ours[-1].name}" if ours else None + ) + return { + "error_type": type(error).__name__, + "error_code": getattr(cause, "pgcode", None), + "where": where, + } + + +def is_inside_transaction() -> bool: + """Whether the main database is already inside a transaction (an atomic block).""" + return bool(transaction.get_connection().in_atomic_block) + + +def has_indexable_file(resource: Resource) -> bool: + """Whether the resource has a stored file in a format that gets a table.""" + file_details = getattr(resource, "resourcefiledetails", None) + if not file_details or not file_details.file: + logger.info("data_index_skipped_no_file", resource_id=str(resource.id)) + return False + file_format = (file_details.format or "").lower() + if file_format not in INDEXED_FORMATS: + logger.info("data_index_skipped_format", resource_id=str(resource.id), format=file_format) + return False + return True + + +def load_resource_frame(resource: Resource) -> Optional[LoadedFile]: + """Read the resource's stored file into a frame and note which file it was. + + Column names are cleaned the way tables have always needed them: double + quotes removed, line breaks and tabs turned into spaces, and names that + then collide get "_1", "_2", ... suffixes. + + Returns None when the file can't be read, takes longer than + LOAD_TIMEOUT_SECONDS, or has no rows (each case logged). + """ + file_details = resource.resourcefiledetails + identity = FileIdentity(file_details.file.name, file_details.sha256) + try: + with local_copy(file_details.file) as path: + frame = _load_with_timeout(path, file_details.format.lower()) + except Exception as error: + logger.error("data_index_load_failed", resource_id=str(resource.id), **error_fields(error)) + return None + if frame is None: + logger.error("data_index_load_failed", resource_id=str(resource.id), error_type="NoFrame") + return None + if frame.empty: + logger.info("data_index_skipped_empty", resource_id=str(resource.id)) + return None + frame.columns = pd.Index(_clean_column_names(list(frame.columns))) + return LoadedFile(frame, identity) + + +def _load_with_timeout(path: str, file_format: str) -> Optional[pd.DataFrame]: + # SIGALRM only works in a Unix process's main thread; elsewhere (a server + # worker thread) the file loads without a limit, as it always has. + if threading.current_thread() is not threading.main_thread() or not hasattr(signal, "SIGALRM"): + return load_tabular_data(path, file_format) + + def on_alarm(signum: int, frame: Any) -> None: + raise TimeoutError(f"loading took longer than {LOAD_TIMEOUT_SECONDS} seconds") + + previous = signal.signal(signal.SIGALRM, on_alarm) + signal.alarm(LOAD_TIMEOUT_SECONDS) + try: + return load_tabular_data(path, file_format) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + + +def _clean_column_names(columns: List[Any]) -> List[str]: + cleaned = [str(c).replace('"', "").replace("\n", " ").replace("\t", " ") for c in columns] + seen: Counter = Counter() + unique = [] + for name in cleaned: + unique.append(f"{name}_{seen[name]}" if seen[name] else name) + seen[name] += 1 + return unique + + +def stored_file_identity(resource: Resource) -> Optional[FileIdentity]: + """The resource's current stored file, read fresh from the database.""" + row = ( + ResourceFileDetails.objects.filter(resource_id=resource.pk) + .values_list("file", "sha256") + .first() + ) + return FileIdentity(*row) if row else None + + +def new_data_table_row(resource: Resource) -> ResourceDataTable: + """An unsaved ResourceDataTable row for this upload, already carrying its table name.""" + row = ResourceDataTable(resource=resource) + row.table_name = data_table_name(row.id) + return row + + +def describe_data_table(resource: Resource, built_at: datetime) -> str: + """The comment stored on a new table, so people browsing the data database know what it holds. + + JSON with the resource id and name, the dataset title and the build time; + the migration command reads ``built_at`` to leave freshly built tables alone. + """ + return json.dumps( + { + "resource_id": str(resource.id), + "resource": resource.name, + "dataset": resource.dataset.title, + "built_at": built_at.isoformat(), + }, + ensure_ascii=False, + ) + + +def build_data_table(table_name: str, frame: pd.DataFrame, comment: str) -> None: + """Create a table holding every row of the frame, in one data-database transaction. + + Creates the table, copies the rows in through a temporary table, and + stores the comment. Raises on any failure, and then nothing is left + behind: a killed process or a bad value mid-copy rolls the whole table back. + """ + table = pg_sql.Identifier(table_name) + staging = pg_sql.Identifier(f"tmp_{table_name}") + columns = pg_sql.SQL(", ").join( + pg_sql.SQL("{} {}").format( + pg_sql.Identifier(str(col)), pg_sql.SQL(get_sql_type(str(frame[col].dtype))) + ) + for col in frame.columns + ) + rows = StringIO() + frame.to_csv(rows, index=False, header=False) + rows.seek(0) + + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute(pg_sql.SQL("CREATE TABLE {} ({})").format(table, columns)) + cursor.execute( + pg_sql.SQL("CREATE TEMP TABLE {} (LIKE {}) ON COMMIT DROP").format(staging, table) + ) + cursor.copy_expert(pg_sql.SQL("COPY {} FROM STDIN WITH CSV").format(staging), rows) + cursor.execute(pg_sql.SQL("INSERT INTO {} SELECT * FROM {}").format(table, staging)) + cursor.execute( + pg_sql.SQL("COMMENT ON TABLE {} IS {}").format(table, pg_sql.Literal(comment)) + ) + + +def lock_resource(resource: Resource) -> None: + """Lock the resource's row until the surrounding transaction ends. + + Two uploads of one resource then switch one after the other. A no-op on + SQLite (tests); Postgres in production. + """ + Resource.objects.select_for_update().only("id").get(pk=resource.pk) + + +def retire_data_table_row(resource: Resource) -> Optional[datetime]: + """Delete the resource's current ResourceDataTable row, returning when it was created. + + Its table is dropped by the delete receiver once the surrounding + transaction commits. Returns None when the resource had no row. + """ + old = ResourceDataTable.objects.filter(resource_id=resource.pk).first() + if old is None: + return None + created = old.created + old.delete() + return created + + +def save_data_table_row(row: ResourceDataTable, created: Optional[datetime]) -> None: + """Save the new row, keeping the retired row's creation time when there was one. + + The data API picks a dataset's default table by creation time, so a + re-upload keeps the dataset's default where it was. + """ + row.save(force_insert=True) + if created is not None: + ResourceDataTable.objects.filter(pk=row.pk).update(created=created) + row.created = created + + +def rebuild_resource_schema(resource: Resource, frame: pd.DataFrame) -> None: + """Replace the resource's column rows with one per frame column. + + Each column's type comes from the frame. Descriptions people wrote are + carried over to columns with the same name; new columns get + "Description of column ". + + Errors escape: inside the switch transaction a swallowed database error + would silently roll the whole switch back. A header longer than the column + row allows is refused up front (ValueError naming its position), the same + outcome Postgres would give, on every database. + """ + max_length = ResourceSchema._meta.get_field("field_name").max_length or 255 + for position, column in enumerate(frame.columns, start=1): + if len(str(column)) > max_length: + raise ValueError(f"column {position} header is longer than {max_length} characters") + + existing = ResourceSchema.objects.filter(resource=resource) + descriptions = { + name: description + for name, description in existing.values_list("field_name", "description") + if description is not None + } + existing.delete() + for column in frame.columns: + sql_type = get_sql_type(str(frame[column].dtype)) + ResourceSchema.objects.create( + resource=resource, + field_name=column, + format=_FIELD_FORMATS.get(sql_type, "STRING"), + description=descriptions.get(column, f"Description of column {column}"), + ) + + +def data_table_row_exists(row_id: Any) -> bool: + """Whether a ResourceDataTable row with this id is saved.""" + return ResourceDataTable.objects.filter(pk=row_id).exists() + + +def drop_data_table(table_name: str) -> bool: + """Drop a table from the data database; never raises. + + Waits at most DROP_LOCK_TIMEOUT for readers holding the table. A failure + is logged with the table name and swallowed — the table is left behind as + an orphan for ``migrate_data_table_names`` to report. Returns whether the + drop went through (a table that was already gone counts as dropped). + """ + try: + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL lock_timeout = {}").format(pg_sql.Literal(DROP_LOCK_TIMEOUT)) + ) + cursor.execute( + pg_sql.SQL("DROP TABLE IF EXISTS {}").format(pg_sql.Identifier(table_name)) + ) + except Exception as error: + logger.error("data_table_drop_failed", table_name=table_name, **error_fields(error)) + return False + logger.info("data_table_dropped", table_name=table_name) + return True + + +def drop_data_table_if_unshared(table_name: str, row_id: Any) -> None: + """Drop a deleted row's table, unless another row still uses the same physical table. + + Only names from before per-upload tables can share one (Postgres cut two + long names to the same 63 bytes); for those, the table stays until its last + row is gone. Query: ``table_name LIKE '%'`` on the unique + table_name index — a sequential scan on some collations, over one row per + tabular resource. Never raises. + """ + try: + shared = may_share_physical_table(table_name) and _another_row_uses(table_name, row_id) + except Exception as error: + logger.error("data_table_drop_failed", table_name=table_name, **error_fields(error)) + return + if shared: + logger.info("data_table_drop_skipped_shared", table_name=table_name, row_id=str(row_id)) + return + drop_data_table(table_name) + + +def _another_row_uses(table_name: str, row_id: Any) -> bool: + physical = physical_table_name(table_name) + candidates = ( + ResourceDataTable.objects.filter(table_name__startswith=physical) + .exclude(pk=row_id) + .values_list("table_name", flat=True) + ) + return any(physical_table_name(name) == physical for name in candidates) diff --git a/pytest.ini b/pytest.ini index b493e2f9..634db965 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,6 @@ python_functions = test_* testpaths = tests pythonpath = . django_find_project = true -addopts = -v --nomigrations --tb=short \ No newline at end of file +addopts = -v --nomigrations --tb=short +markers = + data_db: builds tables in the Postgres data database; needs TEST_DATA_DB_URL (see tests/data_index/conftest.py) diff --git a/tests/data_index/__init__.py b/tests/data_index/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data_index/conftest.py b/tests/data_index/conftest.py new file mode 100644 index 00000000..7703b854 --- /dev/null +++ b/tests/data_index/conftest.py @@ -0,0 +1,111 @@ +"""Shared setup for the data-index tests. + +Tests marked ``data_db`` build real tables in a Postgres data database. They +need TEST_DATA_DB_URL (see tests/test_settings.py), for example: + + docker run -d --name dataspace-test-data-db -e POSTGRES_PASSWORD=postgres \ + -p 55432:5432 postgres:16 + export TEST_DATA_DB_URL=postgres://postgres:postgres@localhost:55432/postgres + +Without it they are skipped, unless REQUIRE_DATA_DB_TESTS=1 (set in CI), in +which case the run fails: a green run that skipped them proves nothing. + +The main database stays SQLite, so row locks are no-ops here; the lock order is +proven by the flow tests and its effect by the journey script. +""" + +import os +import types +from typing import Any, Callable, List, Optional + +import pytest +from django.conf import settings +from django.core.files.base import ContentFile +from django.db import connections + +from api.models import Dataset, Organization, Resource, ResourceFileDetails +from authorization.models import User +from tests.data_index.support import ( + DATA_DB, + fixture_bytes, + list_public_tables, + table_comment, + table_exists, + table_rows, +) + + +def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item]) -> None: + if DATA_DB in settings.DATABASES: + return + data_db_items = [item for item in items if item.get_closest_marker("data_db")] + if not data_db_items: + return + if os.environ.get("REQUIRE_DATA_DB_TESTS") == "1": + raise pytest.UsageError( + f"{len(data_db_items)} data_db tests would be skipped: set TEST_DATA_DB_URL " + "(REQUIRE_DATA_DB_TESTS=1 forbids skipping them)." + ) + skip = pytest.mark.skip(reason="needs TEST_DATA_DB_URL (a Postgres data database)") + for item in data_db_items: + item.add_marker(skip) + + +@pytest.fixture(autouse=True) +def empty_data_db_after_test(request: pytest.FixtureRequest, django_db_blocker: Any) -> Any: + """Drop every table a data_db test left behind. + + pytest-django only empties tables of models routed to a database, and none + are routed to the data database, so without this each test would see the + tables of the tests before it. + """ + yield + if not request.node.get_closest_marker("data_db") or DATA_DB not in settings.DATABASES: + return + with django_db_blocker.unblock(): + for name in list_public_tables(): + with connections[DATA_DB].cursor() as cursor: + cursor.execute(f'DROP TABLE IF EXISTS "{name}" CASCADE') + + +@pytest.fixture +def tables() -> types.SimpleNamespace: + """Read-only peeks into the test data database.""" + return types.SimpleNamespace( + names=list_public_tables, exists=table_exists, rows=table_rows, comment=table_comment + ) + + +@pytest.fixture +def owner(db: Any) -> User: + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def dataset(owner: User) -> Dataset: + org = Organization.objects.create(name="Org", description="o", slug="org") + return Dataset.objects.create( + title="District health indicators", + slug="dhi", + description="d", + user=owner, + organization=org, + ) + + +@pytest.fixture +def add_resource(dataset: Dataset) -> Callable[..., Resource]: + """Create a resource holding a CSV file (fixture file name or raw bytes).""" + + def add(name: str, csv: Any, in_dataset: Optional[Dataset] = None) -> Resource: + content = fixture_bytes(csv) if isinstance(csv, str) else csv + resource = Resource.objects.create(name=name, dataset=in_dataset or dataset) + ResourceFileDetails.objects.create( + resource=resource, + file=ContentFile(content, name=f"{resource.id.hex}.csv"), + size=len(content), + format="csv", + ) + return resource + + return add diff --git a/tests/data_index/fixtures/districts_v1.csv b/tests/data_index/fixtures/districts_v1.csv new file mode 100644 index 00000000..1cb01ba4 --- /dev/null +++ b/tests/data_index/fixtures/districts_v1.csv @@ -0,0 +1,4 @@ +district,cases +Kamrup,10 +Nagaon,20 +Jorhat,30 diff --git a/tests/data_index/fixtures/districts_v2.csv b/tests/data_index/fixtures/districts_v2.csv new file mode 100644 index 00000000..e02f0ba8 --- /dev/null +++ b/tests/data_index/fixtures/districts_v2.csv @@ -0,0 +1,5 @@ +district,cases +Kamrup,11 +Nagaon,22 +Jorhat,33 +Dibrugarh,44 diff --git a/tests/data_index/fixtures/long_header.csv b/tests/data_index/fixtures/long_header.csv new file mode 100644 index 00000000..15305323 --- /dev/null +++ b/tests/data_index/fixtures/long_header.csv @@ -0,0 +1,3 @@ +district,Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anae +Kamrup,1 +Nagaon,2 diff --git a/tests/data_index/fixtures/out_of_range.csv b/tests/data_index/fixtures/out_of_range.csv new file mode 100644 index 00000000..f8d6615a --- /dev/null +++ b/tests/data_index/fixtures/out_of_range.csv @@ -0,0 +1,3 @@ +district,count +Kamrup,1 +Nagaon,18446744073709551615 diff --git a/tests/data_index/fixtures/percent_header.csv b/tests/data_index/fixtures/percent_header.csv new file mode 100644 index 00000000..1b9e7886 --- /dev/null +++ b/tests/data_index/fixtures/percent_header.csv @@ -0,0 +1,4 @@ +district,Literacy rate (%),year +Kamrup,81.5,2021 +Nagaon,72.0,2021 +Jorhat,85.25,2022 diff --git a/tests/data_index/routers.py b/tests/data_index/routers.py new file mode 100644 index 00000000..56d4dcf7 --- /dev/null +++ b/tests/data_index/routers.py @@ -0,0 +1,18 @@ +"""Keep Django's own tables out of the test data database. + +In production the data database holds only data tables (one per tabular +upload). The router mirrors that in tests, so pytest-django creates no model +tables there and an orphan or "is this the main database?" check sees what +production would see. +""" + +from typing import Any, Optional + + +class DataDbTestRouter: + def allow_migrate( + self, db: str, app_label: str, model_name: Optional[str] = None, **hints: Any + ) -> Optional[bool]: + if db == "data_db": + return False + return None diff --git a/tests/data_index/support.py b/tests/data_index/support.py new file mode 100644 index 00000000..d9a04ef0 --- /dev/null +++ b/tests/data_index/support.py @@ -0,0 +1,60 @@ +"""Helpers shared by the data-index tests (fixtures live in conftest.py).""" + +from pathlib import Path +from typing import Any, List, Optional + +import pytest +from django.core.files.base import ContentFile +from django.db import connections + +from api.models import Resource + +FIXTURES = Path(__file__).parent / "fixtures" +DATA_DB = "data_db" + + +def uses_data_db(test: Any) -> Any: + """Mark a test that builds tables in the data database. + + It needs real commits (to see on_commit drops and the switch) and both + databases. Skipped without TEST_DATA_DB_URL; see conftest.py. + """ + test = pytest.mark.django_db(transaction=True, databases=["default", DATA_DB])(test) + return pytest.mark.data_db(test) + + +def list_public_tables() -> List[str]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + return sorted(row[0] for row in cursor.fetchall()) + + +def table_exists(name: str) -> bool: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT to_regclass(%s) IS NOT NULL", [f'public."{name}"']) + return bool(cursor.fetchone()[0]) + + +def table_rows(name: str, order_by: str = "1") -> List[tuple]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute(f'SELECT * FROM "{name}" ORDER BY {order_by}') + return cursor.fetchall() + + +def table_comment(name: str) -> Optional[str]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT obj_description(to_regclass(%s), 'pg_class')", [f'public."{name}"']) + return cursor.fetchone()[0] + + +def fixture_bytes(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def replace_file(resource: Resource, csv: Any) -> None: + """Save a new file for the resource, as an upload request does before indexing.""" + content = fixture_bytes(csv) if isinstance(csv, str) else csv + details = resource.resourcefiledetails + details.file = ContentFile(content, name=f"{resource.id.hex}.csv") + details.size = len(content) + details.save() diff --git a/tests/data_index/test_delete_cleanup.py b/tests/data_index/test_delete_cleanup.py new file mode 100644 index 00000000..fd19926c --- /dev/null +++ b/tests/data_index/test_delete_cleanup.py @@ -0,0 +1,181 @@ +"""Deleting a data-table row drops its table once the delete commits. + +Root cause this guards against: deleting a resource or a dataset never removed +its data tables, so the data database kept every table ever built. The one +receiver on the row's delete also covers re-uploads and admin deletes. +""" + +import os +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import psycopg2 +from django.db import connections, transaction +from structlog.testing import capture_logs + +from api.models import ResourceDataTable +from api.signals.data_table_signals import drop_table_after_row_delete +from api.utils.data_indexing import index_resource_data +from api.utils.data_tables import drop_data_table_if_unshared +from tests.data_index.support import uses_data_db + +REPO_ROOT = Path(__file__).resolve().parents[2] +SHARED_PREFIX = "district-wise-prevalence-of-anaemia-among-women-aged-15-49-dist" # 63 bytes + + +def create_legacy_table(name: str) -> None: + with connections["data_db"].cursor() as cursor: + cursor.execute(f'CREATE TABLE "{name}" (district text)') + + +@uses_data_db +def test_deleting_a_resource_drops_its_table(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + + resource.delete() + + assert not tables.exists(row.table_name) + + +@uses_data_db +def test_deleting_a_dataset_drops_every_table_of_its_resources( + add_resource: Any, dataset: Any, tables: Any +) -> None: + index_resource_data(add_resource("first.csv", "districts_v1.csv")) + index_resource_data(add_resource("second.csv", "districts_v2.csv")) + assert len(tables.names()) == 2 + + dataset.delete() + + assert tables.names() == [] + + +@uses_data_db +def test_a_rolled_back_delete_keeps_the_table(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + + try: + with transaction.atomic(): + resource.delete() + raise RuntimeError("the request failed after the delete") + except RuntimeError: + pass + + assert ResourceDataTable.objects.filter(pk=row.pk).exists() + assert tables.exists(row.table_name) + + +@uses_data_db +def test_a_legacy_table_two_rows_share_is_dropped_only_with_the_last_row( + add_resource: Any, tables: Any +) -> None: + first = ResourceDataTable.objects.create( + resource=add_resource("first.csv", "districts_v1.csv"), + table_name=SHARED_PREFIX + "rict-health-indicators-nfhs-4_1a2b3c4d", + ) + second = ResourceDataTable.objects.create( + resource=add_resource("second.csv", "districts_v1.csv"), + table_name=SHARED_PREFIX + "rict-health-indicators-nfhs-5_5e6f7a8b", + ) + create_legacy_table(first.table_name) + assert tables.names() == [SHARED_PREFIX] + + with capture_logs() as logs: + first.delete() + assert tables.names() == [SHARED_PREFIX] + assert any(log["event"] == "data_table_drop_skipped_shared" for log in logs) + + second.delete() + assert tables.names() == [] + + +@uses_data_db +def test_dropping_a_table_that_is_already_gone_does_not_fail_the_delete( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + ResourceDataTable.objects.create(resource=resource, table_name="rdt_never_built") + + resource.delete() + + assert not ResourceDataTable.objects.filter(resource_id=resource.pk).exists() + + +@uses_data_db +def test_a_drop_blocked_by_a_reader_is_logged_and_the_delete_still_succeeds( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + settings = connections["data_db"].settings_dict + reader = psycopg2.connect( + dbname=settings["NAME"], + user=settings["USER"], + password=settings["PASSWORD"], + host=settings["HOST"], + port=settings["PORT"], + ) + try: + with reader.cursor() as cursor: + cursor.execute(f'LOCK TABLE "{row.table_name}" IN ACCESS SHARE MODE') + + with patch("api.utils.data_tables.DROP_LOCK_TIMEOUT", "200ms"), capture_logs() as logs: + resource.delete() + finally: + reader.rollback() + reader.close() + + assert not ResourceDataTable.objects.filter(pk=row.pk).exists() + assert tables.exists(row.table_name) + failures = [log for log in logs if log["event"] == "data_table_drop_failed"] + assert failures and failures[0]["table_name"] == row.table_name + + +def test_the_receiver_schedules_one_guarded_drop_per_deleted_row(add_resource: Any) -> None: + first = ResourceDataTable.objects.create(resource=add_resource("a.csv", "districts_v1.csv")) + second = ResourceDataTable.objects.create(resource=add_resource("b.csv", "districts_v1.csv")) + expected = [(first.table_name, first.pk), (second.table_name, second.pk)] + + with patch("api.signals.data_table_signals.transaction.on_commit") as on_commit: + ResourceDataTable.objects.filter(pk__in=[first.pk, second.pk]).delete() + + assert on_commit.call_count == 2 + scheduled = [] + for call in on_commit.call_args_list: + callback = call.args[0] + assert isinstance(callback, partial) + assert callback.func is drop_data_table_if_unshared + assert call.kwargs == {"using": "default", "robust": True} + scheduled.append(callback.args) + assert sorted(scheduled) == sorted(expected) + + +def test_the_receiver_is_connected_once_the_app_loads() -> None: + # A fresh interpreter: an import elsewhere in this test run can't connect it for us. + check = ( + "import django; django.setup()\n" + "from django.db.models.signals import post_delete\n" + "from api.models import ResourceDataTable\n" + "receivers = post_delete._live_receivers(ResourceDataTable)[0]\n" + "names = {(r.__module__, r.__name__) for r in receivers}\n" + "print(('api.signals.data_table_signals', 'drop_table_after_row_delete') in names)\n" + ) + env = {**os.environ, "DJANGO_SETTINGS_MODULE": "tests.test_settings"} + env.pop("TEST_DATA_DB_URL", None) + result = subprocess.run( + [sys.executable, "-c", check], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr[-2000:] + assert result.stdout.strip().splitlines()[-1] == "True" + assert drop_table_after_row_delete.__module__ == "api.signals.data_table_signals" diff --git a/tests/data_index/test_reader_regression.py b/tests/data_index/test_reader_regression.py new file mode 100644 index 00000000..152a8b4a --- /dev/null +++ b/tests/data_index/test_reader_regression.py @@ -0,0 +1,59 @@ +"""Every reader of data tables still works on the new per-upload tables. + +The readers look the table up from the resource's ResourceDataTable row, so +the new names should be invisible to them. This proves it for each reader, +on a header with a percent sign (common in indicator tables). +""" + +from typing import Any + +from api.models import ResourcePreviewDetails, ResourceSchema +from api.utils.data_indexing import ( + fetch_resource_data, + get_preview_data, + get_resource_columns, + get_row_count, + index_resource_data, + query_resource_data, +) +from tests.data_index.support import uses_data_db + +COLUMNS = ["district", "Literacy rate (%)", "year"] + + +@uses_data_db +def test_every_reader_works_on_a_new_table_with_a_percent_header(add_resource: Any) -> None: + resource = add_resource("literacy.csv", "percent_header.csv") + resource.preview_enabled = True + resource.preview_details = ResourcePreviewDetails.objects.create(is_all_entries=True) + resource.save() + row = index_resource_data(resource) + assert row is not None and row.table_name.startswith("rdt_") + + assert get_row_count(resource) == 3 + + preview = get_preview_data(resource) + assert preview is not None + assert preview.columns == COLUMNS + assert len(preview.rows) == 3 + + assert get_resource_columns(resource) == COLUMNS + + fetched = fetch_resource_data( + resource, + filters={"Literacy rate (%)__gte": 80}, + columns=["district", "Literacy rate (%)"], + order_by=["-Literacy rate (%)"], + ) + assert fetched["rows"] == [["Jorhat", 85.25], ["Kamrup", 81.5]] + assert fetched["total"] == 2 + + chart = query_resource_data( + resource, 'SELECT district, "Literacy rate (%)" FROM {{table}} WHERE year = 2022' + ) + assert chart is not None + assert chart.values.tolist() == [["Jorhat", 85.25]] + + # With no column rows, the column list falls back to the table itself. + ResourceSchema.objects.filter(resource=resource).delete() + assert get_resource_columns(resource) == COLUMNS diff --git a/tests/data_index/test_reindex_switch.py b/tests/data_index/test_reindex_switch.py new file mode 100644 index 00000000..52b9f13d --- /dev/null +++ b/tests/data_index/test_reindex_switch.py @@ -0,0 +1,434 @@ +"""Re-uploads build the new table first and switch over in one commit. + +Root causes this guards against: +- The old table was dropped before the new one existed, so readers saw empty or + half-written data, and a failed re-upload lost the data for good. +- The column-row rebuild swallowed errors per column. Inside the switch's + transaction one failure silently rolled everything back while the upload + still reported success. +- Two overlapping uploads of one resource each indexed their own file, and the + slower (older) one could switch in last, so the download and the served rows + disagreed. +""" + +from datetime import datetime, timezone +from typing import Any, List +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +from django.db import connections, transaction +from django.db.utils import ProgrammingError +from rest_framework.test import APIClient +from structlog.testing import capture_logs + +from api.models import Resource, ResourceDataTable, ResourceSchema +from api.utils import data_tables +from api.utils.data_indexing import index_resource_data +from api.utils.data_tables import ( + FileIdentity, + LoadedFile, + has_indexable_file, + load_resource_frame, + rebuild_resource_schema, +) +from tests.data_index.support import fixture_bytes, replace_file, uses_data_db + +# --- Layer 1: the real flow against both databases -------------------------- + + +@uses_data_db +def test_first_upload_creates_a_named_table_with_its_columns( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + + row = index_resource_data(resource) + + assert row is not None and row.table_name == "rdt_" + row.id.hex + assert ResourceDataTable.objects.get(resource=resource).table_name == row.table_name + assert tables.rows(row.table_name) == [("Jorhat", 30), ("Kamrup", 10), ("Nagaon", 20)] + assert list( + ResourceSchema.objects.filter(resource=resource) + .order_by("id") + .values_list("field_name", "format") + ) == [("district", "STRING"), ("cases", "INTEGER")] + assert str(resource.id) in tables.comment(row.table_name) + assert "cases.csv" in tables.comment(row.table_name) + + +@uses_data_db +def test_reupload_switches_to_a_new_table_and_drops_the_old_one( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + ResourceSchema.objects.filter(resource=resource, field_name="cases").update( + description="Confirmed cases" + ) + + replace_file(resource, "districts_v2.csv") + new = index_resource_data(resource) + + assert new is not None and new.table_name != old.table_name + assert ResourceDataTable.objects.get(resource=resource).pk == new.pk + assert len(tables.rows(new.table_name)) == 4 + assert not tables.exists(old.table_name) + assert tables.names() == [new.table_name] + assert ( + ResourceSchema.objects.get(resource=resource, field_name="cases").description + == "Confirmed cases" + ) + + +@uses_data_db +def test_readers_see_the_complete_old_table_until_the_switch_commits( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + seen: dict = {} + real_lock = data_tables.lock_resource + real_drop = data_tables.drop_data_table_if_unshared + + def spy_lock(res: Resource) -> None: + new_tables = [t for t in tables.names() if t != old.table_name] + seen["new_rows_at_switch"] = len(tables.rows(new_tables[0])) + seen["old_exists_at_switch"] = tables.exists(old.table_name) + # The lock only holds inside the switch's transaction (Postgres refuses it outside). + seen["lock_inside_transaction"] = transaction.get_connection().in_atomic_block + real_lock(res) + + def spy_drop(table_name: str, row_id: Any) -> None: + seen["dropped"] = table_name + seen["open_transaction_at_drop"] = transaction.get_connection().in_atomic_block + seen["new_row_saved_at_drop"] = ( + ResourceDataTable.objects.get(resource=resource).table_name != old.table_name + ) + real_drop(table_name, row_id) + + with ( + patch("api.utils.data_indexing.lock_resource", spy_lock), + patch("api.signals.data_table_signals.drop_data_table_if_unshared", spy_drop), + ): + new = index_resource_data(resource) + + assert new is not None + assert seen == { + "new_rows_at_switch": 4, + "old_exists_at_switch": True, + "lock_inside_transaction": True, + "dropped": old.table_name, + "open_transaction_at_drop": False, + "new_row_saved_at_drop": True, + } + + +@uses_data_db +def test_a_failed_build_keeps_the_old_data_and_leaves_no_table( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + + # A count past the BIGINT range: the copy into the new table fails midway. + replace_file(resource, "out_of_range.csv") + with capture_logs() as logs: + assert index_resource_data(resource) is None + + assert ResourceDataTable.objects.get(resource=resource).pk == old.pk + assert tables.names() == [old.table_name] + assert len(tables.rows(old.table_name)) == 3 + # The failure is logged by its code, never by the offending cell's value. + failures = [log for log in logs if log["event"] == "data_index_failed"] + assert failures and failures[0]["error_code"] == "22003" + assert failures[0]["where"].startswith("data_tables.py:") + assert "18446744073709551615" not in repr(logs) + + +@uses_data_db +def test_a_header_too_long_for_the_column_list_fails_the_switch_and_keeps_the_old_data( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + old_columns = list( + ResourceSchema.objects.filter(resource=resource).values_list("field_name", flat=True) + ) + + replace_file(resource, "long_header.csv") + assert index_resource_data(resource) is None + + assert ResourceDataTable.objects.get(resource=resource).pk == old.pk + assert tables.names() == [old.table_name] + assert ( + list(ResourceSchema.objects.filter(resource=resource).values_list("field_name", flat=True)) + == old_columns + ) + + +@uses_data_db +def test_a_reader_still_holding_the_old_name_gets_does_not_exist_not_new_rows( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + stale = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + index_resource_data(resource) + + with pytest.raises(ProgrammingError, match="does not exist"): + with connections["data_db"].cursor() as cursor: + cursor.execute(f'SELECT * FROM "{stale.table_name}"') + + +@uses_data_db +def test_a_slower_build_of_an_older_file_backs_off_when_a_newer_file_was_saved( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + real_build = data_tables.build_data_table + newer: dict = {} + + def build_then_newer_upload_lands(table_name: str, frame: pd.DataFrame, comment: str) -> None: + real_build(table_name, frame, comment) + if newer: + return # the other request's own build + newer["started"] = True + # Meanwhile another request saves a newer file and finishes indexing it. + other_request = Resource.objects.get(pk=resource.pk) + replace_file(other_request, "districts_v2.csv") + newer["row"] = index_resource_data(other_request) + + with patch("api.utils.data_indexing.build_data_table", build_then_newer_upload_lands): + assert index_resource_data(resource) is None + + current = ResourceDataTable.objects.get(resource=resource) + assert current.pk == newer["row"].pk + assert tables.names() == [current.table_name] + assert len(tables.rows(current.table_name)) == 4 + + +@uses_data_db +def test_indexing_inside_an_outer_transaction_is_refused(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + + with capture_logs() as logs, transaction.atomic(): + assert index_resource_data(resource) is None + + assert tables.names() == [] + assert not ResourceDataTable.objects.filter(resource=resource).exists() + assert any(log["event"] == "data_index_refused_inside_transaction" for log in logs) + + +@uses_data_db +def test_reupload_keeps_the_datasets_default_table( + add_resource: Any, owner: Any, dataset: Any +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + first_row = index_resource_data(first) + index_resource_data(second) + + replace_file(first, "districts_v2.csv") + replaced = index_resource_data(first) + + assert replaced is not None and replaced.created == first_row.created + assert ResourceDataTable.objects.get(pk=replaced.pk).created == first_row.created + client = APIClient() + client.force_authenticate(owner) + response = client.get(f"/api/datasets/{dataset.id}/data/") + assert response.status_code == 200, response.content + assert response.json()["resource_id"] == str(first.id) + + +# --- Layer 1: helpers on the main database only ------------------------------ + + +def test_a_resource_without_a_supported_file_is_not_indexable(add_resource: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + assert has_indexable_file(resource) + + resource.resourcefiledetails.format = "pdf" + assert not has_indexable_file(resource) + + bare = Resource.objects.create(name="empty", dataset=resource.dataset) + assert not has_indexable_file(bare) + + +def test_load_remembers_which_stored_file_it_read(add_resource: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + details = resource.resourcefiledetails + + loaded = load_resource_frame(resource) + + assert loaded is not None + assert loaded.file == FileIdentity(details.file.name, details.sha256) + assert list(loaded.frame.columns) == ["district", "cases"] + + +def test_load_returns_nothing_for_a_file_with_only_a_header(add_resource: Any) -> None: + resource = add_resource("cases.csv", b"district,cases\n") + assert load_resource_frame(resource) is None + + +def test_load_cleans_quotes_line_breaks_and_duplicate_headers(add_resource: Any) -> None: + # pandas renames plain duplicates itself; these two only collide once the quote goes. + resource = add_resource("cases.csv", b'"dis""trict","a\nb","x""",x\nKamrup,1,2,3\n') + loaded = load_resource_frame(resource) + assert loaded is not None + assert list(loaded.frame.columns) == ["district", "a b", "x", "x_1"] + + +def test_rebuild_columns_lets_a_database_error_escape(add_resource: Any) -> None: + from django.db import DatabaseError + + resource = add_resource("cases.csv", "districts_v1.csv") + frame = pd.DataFrame({"district": ["Kamrup"], "cases": [1]}) + + def failing_create(**kwargs: Any) -> None: + raise DatabaseError("value too long for type character varying(255)") + + with patch.object(ResourceSchema.objects, "create", failing_create): + with pytest.raises(DatabaseError): + rebuild_resource_schema(resource, frame) + + +def test_rebuild_columns_refuses_a_header_longer_than_the_column_list_allows( + add_resource: Any, +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + frame = pd.DataFrame({"district": ["Kamrup"], "x" * 256: [1]}) + with pytest.raises(ValueError, match="column 2"): + rebuild_resource_schema(resource, frame) + assert not ResourceSchema.objects.filter(resource=resource).exists() + + +# --- Layer 3: the flow's order and clean-up (helpers mocked) ----------------- + +FLOW_HELPERS = [ + "is_inside_transaction", + "has_indexable_file", + "load_resource_frame", + "new_data_table_row", + "describe_data_table", + "build_data_table", + "lock_resource", + "stored_file_identity", + "retire_data_table_row", + "save_data_table_row", + "rebuild_resource_schema", + "data_table_row_exists", + "drop_data_table", +] +FILE = FileIdentity("resources/cases.csv", "abc123") + + +@pytest.fixture +def flow() -> Any: + """Every helper the flow calls, mocked on one parent so call order is recorded.""" + parent = MagicMock() + patchers = [ + patch(f"api.utils.data_indexing.{name}", getattr(parent, name)) for name in FLOW_HELPERS + ] + for p in patchers: + p.start() + parent.is_inside_transaction.return_value = False + parent.has_indexable_file.return_value = True + parent.load_resource_frame.return_value = LoadedFile(pd.DataFrame({"a": [1]}), FILE) + parent.new_data_table_row.return_value = MagicMock(table_name="rdt_new", pk="new-id") + parent.stored_file_identity.return_value = FILE + parent.retire_data_table_row.return_value = datetime(2026, 1, 1, tzinfo=timezone.utc) + parent.data_table_row_exists.return_value = True + yield parent + for p in patchers: + p.stop() + + +def called(flow: Any) -> List[str]: + return [c[0] for c in flow.mock_calls if "." not in c[0]] + + +def test_flow_builds_then_locks_before_retiring_the_old_row(flow: Any) -> None: + row = index_resource_data(MagicMock()) + + assert row is flow.new_data_table_row.return_value + assert called(flow) == [ + "is_inside_transaction", + "has_indexable_file", + "load_resource_frame", + "new_data_table_row", + "describe_data_table", + "build_data_table", + "lock_resource", + "stored_file_identity", + "retire_data_table_row", + "save_data_table_row", + "rebuild_resource_schema", + "data_table_row_exists", + ] + flow.save_data_table_row.assert_called_once_with( + flow.new_data_table_row.return_value, created=flow.retire_data_table_row.return_value + ) + + +def raise_error(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("boom") + + +@pytest.mark.parametrize( + "helper", + ["build_data_table", "lock_resource", "retire_data_table_row", "rebuild_resource_schema"], +) +def test_flow_drops_its_table_when_a_step_fails(flow: Any, helper: str) -> None: + getattr(flow, helper).side_effect = raise_error + + assert index_resource_data(MagicMock()) is None + + flow.drop_data_table.assert_called_once_with("rdt_new") + + +def test_flow_drops_its_table_when_a_newer_file_was_saved(flow: Any) -> None: + flow.stored_file_identity.return_value = FileIdentity("resources/newer.csv", "def456") + + assert index_resource_data(MagicMock()) is None + + flow.retire_data_table_row.assert_not_called() + flow.drop_data_table.assert_called_once_with("rdt_new") + + +def test_flow_drops_its_table_when_the_new_row_is_missing_after_the_commit(flow: Any) -> None: + flow.data_table_row_exists.return_value = False + + assert index_resource_data(MagicMock()) is None + + flow.drop_data_table.assert_called_once_with("rdt_new") + + +def test_flow_drops_its_table_even_when_interrupted(flow: Any) -> None: + flow.build_data_table.side_effect = KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + index_resource_data(MagicMock()) + + flow.drop_data_table.assert_called_once_with("rdt_new") + + +def test_flow_keeps_the_table_after_a_confirmed_switch(flow: Any) -> None: + index_resource_data(MagicMock()) + flow.drop_data_table.assert_not_called() + + +@pytest.mark.parametrize("stop_at", ["inside_transaction", "no_file", "nothing_loaded"]) +def test_flow_builds_nothing_when_there_is_nothing_to_index(flow: Any, stop_at: str) -> None: + if stop_at == "inside_transaction": + flow.is_inside_transaction.return_value = True + elif stop_at == "no_file": + flow.has_indexable_file.return_value = False + else: + flow.load_resource_frame.return_value = None + + assert index_resource_data(MagicMock()) is None + + flow.build_data_table.assert_not_called() + flow.drop_data_table.assert_not_called() diff --git a/tests/data_index/test_table_names.py b/tests/data_index/test_table_names.py new file mode 100644 index 00000000..d2c98ceb --- /dev/null +++ b/tests/data_index/test_table_names.py @@ -0,0 +1,93 @@ +"""Every tabular upload gets its own table, named from its own row id. + +Root cause this guards against: table names used to be built from the resource +name and dataset title (up to 110 characters). Postgres keeps only the first 63 +bytes of a name, so two resources with long, similar names shared one table and +a re-upload of one replaced the other's data. +""" + +import json +import uuid +from datetime import datetime, timezone + +from api.models import ResourceDataTable +from api.utils.data_indexing import index_resource_data +from api.utils.data_table_names import data_table_name, physical_table_name +from api.utils.data_tables import build_data_table, describe_data_table +from tests.data_index.support import uses_data_db + +LEGACY_NAME = ( + "district-wise-prevalence-of-anaemia-among-women-aged-15-49-" + "district-health-indicators-assam-nfhs-5_1a2b3c4d" +) + + +def test_a_table_name_is_rdt_and_the_row_id_in_lowercase_hex() -> None: + row_id = uuid.UUID("0123456789ABCDEF0123456789ABCDEF") + assert data_table_name(row_id) == "rdt_0123456789abcdef0123456789abcdef" + assert len(data_table_name(row_id)) == 36 + + +def test_physical_name_is_cut_to_63_bytes_like_postgres() -> None: + assert len(LEGACY_NAME) > 100 + assert physical_table_name(LEGACY_NAME) == LEGACY_NAME[:63] + short = data_table_name(uuid.uuid4()) + assert physical_table_name(short) == short + + +def test_physical_name_never_cuts_a_character_in_half() -> None: + # 62 ASCII bytes then a 2-byte character: Postgres keeps 62 bytes, not 63. + name = "a" * 62 + "é" + "tail" + assert physical_table_name(name) == "a" * 62 + + +def test_a_new_row_is_named_from_its_own_id(add_resource: object) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = ResourceDataTable.objects.create(resource=resource) + assert row.table_name == "rdt_" + row.id.hex + + +def test_an_existing_row_keeps_its_stored_name_on_save(add_resource: object) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = ResourceDataTable.objects.create(resource=resource, table_name=LEGACY_NAME) + row.save() + row.refresh_from_db() + assert row.table_name == LEGACY_NAME + + +@uses_data_db +def test_two_resources_with_long_similar_names_get_their_own_tables( + add_resource: object, tables: object +) -> None: + prefix = "District wise prevalence of anaemia among women aged 15 to 49 " + first = add_resource(prefix + "(NFHS-4)", "districts_v1.csv") + second = add_resource(prefix + "(NFHS-5)", "districts_v2.csv") + + first_row = index_resource_data(first) + second_row = index_resource_data(second) + + assert first_row is not None and second_row is not None + assert first_row.table_name != second_row.table_name + assert [r[1] for r in tables.rows(first_row.table_name)] == [30, 10, 20] + assert len(tables.rows(second_row.table_name)) == 4 + + +@uses_data_db +def test_the_table_comment_is_stored_as_text_not_run_as_sql( + add_resource: object, tables: object +) -> None: + import pandas as pd + + resource = add_resource("cases'; DROP TABLE x; --\"", "districts_v1.csv") + built_at = datetime(2026, 11, 13, 10, 30, tzinfo=timezone.utc) + comment = describe_data_table(resource, built_at=built_at) + + build_data_table("rdt_comment_check", pd.DataFrame({"a": [1]}), comment) + + stored = json.loads(tables.comment("rdt_comment_check")) + assert stored == { + "resource_id": str(resource.id), + "resource": "cases'; DROP TABLE x; --\"", + "dataset": "District health indicators", + "built_at": "2026-11-13T10:30:00+00:00", + } diff --git a/tests/data_index/test_upload_mutations.py b/tests/data_index/test_upload_mutations.py new file mode 100644 index 00000000..c5c9f313 --- /dev/null +++ b/tests/data_index/test_upload_mutations.py @@ -0,0 +1,128 @@ +"""Upload, replace and delete through the GraphQL API, read through the data API. + +Who may call these mutations is unchanged and covered by +tests/schema/test_resource_permissions.py; these tests follow a dataset owner +through the data-table lifecycle end to end. +""" + +import types +from typing import Any, Dict + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory +from rest_framework.test import APIClient + +from api.models import Resource, ResourceDataTable +from api.schema.schema import schema +from tests.data_index.support import fixture_bytes, uses_data_db + +CREATE = """mutation ($ds: UUID!, $file: Upload!) { + createFileResources(fileResourceInput: {dataset: $ds, files: [$file]}) { id } +}""" +REPLACE = """mutation ($id: UUID!, $file: Upload!) { + updateFileResource(fileResourceInput: {id: $id, file: $file}) { + __typename ... on OperationInfo { messages { message } } + } +}""" +RESET = """mutation ($id: UUID!) { + resetFileResourceSchema(resourceId: $id) { + __typename ... on OperationInfo { messages { message } } + } +}""" +DELETE_RESOURCE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" +DELETE_DATASET = "mutation ($id: UUID!) { deleteDataset(datasetId: $id) }" + + +def run(user: Any, query: str, **variables: Any) -> Dict[str, Any]: + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + result = schema.execute_sync(query, variable_values=variables, context_value=ctx) + assert result.errors is None, result.errors + payload = next(iter(result.data.values())) + if isinstance(payload, dict): + assert payload.get("__typename") != "OperationInfo", payload + return result.data + + +def csv_upload(name: str) -> SimpleUploadedFile: + return SimpleUploadedFile("cases.csv", fixture_bytes(name), content_type="text/csv") + + +def data_api(owner: Any, resource_id: Any) -> Any: + client = APIClient() + client.force_authenticate(owner) + return client.get(f"/api/resources/{resource_id}/data/?order_by=district") + + +def upload(owner: Any, dataset: Any, name: str) -> Resource: + data = run(owner, CREATE, ds=str(dataset.id), file=csv_upload(name)) + return Resource.objects.get(id=data["createFileResources"][0]["id"]) + + +@uses_data_db +def test_an_uploaded_csv_is_served_from_its_own_table( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + + table = ResourceDataTable.objects.get(resource=resource).table_name + assert table.startswith("rdt_") and tables.exists(table) + response = data_api(owner, resource.id) + assert response.status_code == 200, response.content + assert response.json()["rows"] == [["Jorhat", 30], ["Kamrup", 10], ["Nagaon", 20]] + + +@uses_data_db +def test_a_replaced_file_is_served_and_its_old_table_is_gone( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + old_table = ResourceDataTable.objects.get(resource=resource).table_name + + run(owner, REPLACE, id=str(resource.id), file=csv_upload("districts_v2.csv")) + + assert not tables.exists(old_table) + assert tables.names() == [ResourceDataTable.objects.get(resource=resource).table_name] + assert data_api(owner, resource.id).json()["total"] == 4 + + +@uses_data_db +def test_resetting_the_columns_switches_to_a_fresh_table( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + old_table = ResourceDataTable.objects.get(resource=resource).table_name + + run(owner, RESET, id=str(resource.id)) + + new_table = ResourceDataTable.objects.get(resource=resource).table_name + assert new_table != old_table + assert tables.names() == [new_table] + assert data_api(owner, resource.id).json()["total"] == 3 + + +@uses_data_db +def test_deleting_a_resource_removes_its_table_and_its_data( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + + run(owner, DELETE_RESOURCE, id=str(resource.id)) + + assert tables.names() == [] + assert data_api(owner, resource.id).status_code == 404 + + +@uses_data_db +def test_deleting_a_dataset_removes_every_table_it_had( + owner: Any, dataset: Any, tables: Any +) -> None: + upload(owner, dataset, "districts_v1.csv") + upload(owner, dataset, "districts_v2.csv") + assert len(tables.names()) == 2 + + run(owner, DELETE_DATASET, id=str(dataset.id)) + + assert tables.names() == [] diff --git a/tests/test_settings.py b/tests/test_settings.py index 05375f6d..2159f0b0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -33,6 +33,18 @@ } } +# Data tables live in a second Postgres database. Tests that build them +# (marked ``data_db``) run only when TEST_DATA_DB_URL points at a Postgres +# server; pytest-django creates and drops its own database there. The router +# keeps every Django model out of it, as in production. +_TEST_DATA_DB_URL = os.environ.get("TEST_DATA_DB_URL") +if _TEST_DATA_DB_URL: + DATABASES["data_db"] = { + **env.db_url_config(_TEST_DATA_DB_URL), # noqa: F405 — from DataSpace.settings + "TEST": {"NAME": "test_dataspace_data_db"}, + } + DATABASE_ROUTERS = ["tests.data_index.routers.DataDbTestRouter"] + # Disable debug mode DEBUG = True From 33c77550db081edbdb97e99ae54bf54cd314289f Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 06:01:41 +0530 Subject: [PATCH 49/55] feat(data-index): add migrate_data_table_names for existing tables Existing tables still carry title-based names, and two resources can share one (Postgres cut both names to the same 63 bytes), so one of them serves the other's data. This one-off command moves them to rdt_ names. Short old names can't have collided and are renamed in place, holding the resource row and waiting at most 2 s for readers. Long ones are rebuilt from each resource's own file through the normal upload flow, oldest first and with no read-time limit; the shared table is dropped once its last row has moved. A long table whose file is missing or can no longer become a table is renamed and marked "unverified" in its comment if no other row uses it, and every later report lists it until the resource is re-uploaded. A rerun after a crash finishes half-done renames. It is a dry run by default. The dry run checks each stored file, so its report shows what --apply will do, which tables are shared and which rows need a person. Tables no row uses are listed with size and comment. They are dropped only with --apply --drop-orphans --from-report , and only if still unused at that moment. The command refuses to run when the data database looks like the main one. Every row's outcome is logged as it happens, and an in-progress report is written before any change. The Layer 5 journey (re-upload while polling, racing re-uploads, delete) is added for on-demand runs. Plan: plans/claude/data-index-table-names.md (Phase 2) --- .../commands/migrate_data_table_names.py | 166 +++++ api/utils/data_indexing.py | 12 +- api/utils/data_table_migration.py | 475 ++++++++++++ api/utils/data_tables.py | 113 ++- .../test_migrate_data_table_names.py | 690 ++++++++++++++++++ .../data_index/reupload-and-delete.py | 149 ++++ 6 files changed, 1582 insertions(+), 23 deletions(-) create mode 100644 api/management/commands/migrate_data_table_names.py create mode 100644 api/utils/data_table_migration.py create mode 100644 tests/data_index/test_migrate_data_table_names.py create mode 100644 tests/journeys/data_index/reupload-and-delete.py diff --git a/api/management/commands/migrate_data_table_names.py b/api/management/commands/migrate_data_table_names.py new file mode 100644 index 00000000..d48e6a0d --- /dev/null +++ b/api/management/commands/migrate_data_table_names.py @@ -0,0 +1,166 @@ +"""Move existing data tables to per-upload names, and list the tables nothing uses. + + python manage.py migrate_data_table_names # dry run: write a report, change nothing + python manage.py migrate_data_table_names --apply # rename, rebuild, fix rows + python manage.py migrate_data_table_names --apply --drop-orphans --from-report + + --resource-id act on one resource only + --report where to write the JSON report (default: a timestamped file here) + +Short old names are renamed in place; long ones (which may hold another +resource's data) are rebuilt from each resource's own file. A long one whose +file is missing or unusable is renamed and marked "unverified" (listed in +every report until the resource is re-uploaded), or left alone if another row +shares it. A dry run checks the files too, so it shows what --apply will do. +Safe to rerun: each run picks up whatever is left ("nothing to do" once only +rows needing a person remain), and doubles as an orphan audit. + +Docs (runbook, failure model): ../../utils/data_indexing_architecture.md +""" + +from typing import Any, Dict, List + +from django.core.management.base import BaseCommand, CommandError, CommandParser +from django.utils import timezone + +from api.utils.data_table_migration import ( + access_models_touched, + build_report, + drop_reviewed_orphans, + find_orphans, + find_unverified, + nothing_left_to_do, + plan_table_migration, + point_row_at_its_table, + read_data_table_rows, + read_reviewed_orphans, + rebuild_decision, + rebuild_from_file, + record_outcome, + reindex_decision, + reindex_missing_table, + rename_legacy_table, + same_file, + summary_lines, + write_report, +) +from api.utils.data_tables import ( + DataDbNotSeparate, + assert_data_db_is_separate, + list_data_db_tables, +) + + +class Command(BaseCommand): + help = "Move data tables to per-upload rdt_ names; list (and drop reviewed) orphans." + + def add_arguments(self, parser: CommandParser) -> None: + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", help="Report only (the default).") + mode.add_argument("--apply", action="store_true", help="Rename, rebuild and fix rows.") + parser.add_argument( + "--drop-orphans", + action="store_true", + help="With --apply: drop orphans listed in --from-report that are still orphans.", + ) + parser.add_argument("--from-report", help="A dry-run report someone reviewed.") + parser.add_argument("--resource-id", help="Act on this resource only.") + parser.add_argument("--report", help="Where to write this run's JSON report.") + + def handle(self, *args: Any, **options: Any) -> None: + apply, drop_orphans, from_report = ( + options["apply"], + options["drop_orphans"], + options["from_report"], + ) + mode = "apply" if apply else "dry-run" + now = timezone.now() + report_path = options["report"] or f"data_table_migration_{now:%Y%m%dT%H%M%SZ}.json" + + # Orphans are dropped only by an applying run, and only from a reviewed report. + if drop_orphans and not (apply and from_report): + raise CommandError("--drop-orphans needs --apply and --from-report ") + if from_report and same_file(report_path, from_report): + raise CommandError("--report would overwrite the reviewed --from-report file") + + # Never touch anything if the data database might be the main database. + try: + assert_data_db_is_separate() + except DataDbNotSeparate as error: + raise CommandError(f"Refusing to run: {error}") from error + + # Read the reviewed orphan list now, so a bad file stops the run before any change. + reviewed = read_reviewed_orphans(from_report) if drop_orphans else set() + + # Write an in-progress report first: the path works, and a run that dies leaves a trace. + write_report(report_path, build_report(mode, now, status="in_progress")) + + # Read every row and every data table, and work out what each row needs. + plan = plan_table_migration( + read_data_table_rows(), list_data_db_tables(), now, resource_id=options["resource_id"] + ) + entries: List[Dict[str, Any]] = [ + record_outcome(row, "none", "already_migrated") for row in plan.already_migrated + ] + + if apply: + # Finish renames a crashed run left half done: just point the row at its table. + for row in plan.row_updates: + entries.append(record_outcome(row, "row_update", point_row_at_its_table(row))) + + # Rename short old tables in place; they can't have collided. + for row in plan.renames: + entries.append(record_outcome(row, "rename", rename_legacy_table(row))) + + # Rebuild long old tables from each resource's own file, oldest first. + for row, shared in plan.rebuilds: + outcome = rebuild_from_file(row, shared) + entries.append(record_outcome(row, "rebuild", outcome, shared=shared)) + + # Re-index rows whose table is gone, from their file. + for row in plan.missing_tables: + entries.append(record_outcome(row, "reindex", reindex_missing_table(row))) + else: + # A dry run changes nothing, but checks each stored file so the report shows + # what --apply would really do (rebuild, unverified rename, or leave for a person). + for row in plan.row_updates: + entries.append(record_outcome(row, "row_update", "planned_row_update")) + for row in plan.renames: + entries.append(record_outcome(row, "rename", "planned_rename")) + for row, shared in plan.rebuilds: + decision = rebuild_decision(row, shared) + entries.append(record_outcome(row, "rebuild", f"planned_{decision}", shared=shared)) + for row in plan.missing_tables: + entries.append(record_outcome(row, "reindex", f"planned_{reindex_decision(row)}")) + + # Rebuilt resources get new column rows, which resets access models' column choices. + touched = access_models_touched( + [row.resource_id for row, _ in plan.rebuilds] + + [row.resource_id for row in plan.missing_tables] + ) + + dropped: List[str] = [] + if drop_orphans: + # Drop the reviewed orphans that are still orphans right now. + current = find_orphans(read_data_table_rows(), list_data_db_tables(), timezone.now()) + dropped = drop_reviewed_orphans(reviewed, current) + + # List every table still marked unverified, from this run or any earlier one. + unverified = find_unverified(read_data_table_rows(), list_data_db_tables()) + + # Write the full report and print the summary. + report = build_report( + mode, + now, + entries, + plan.orphans, + dropped, + touched, + unverified, + plan.shared_groups, + ) + write_report(report_path, report) + if nothing_left_to_do(entries): + self.stdout.write("nothing to do") + for line in summary_lines(report, report_path): + self.stdout.write(line) diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 63080b41..6a5af5aa 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -23,6 +23,7 @@ from api.utils.data_tables import ( # noqa: F401 — DATA_DB, INDEXED_FORMATS, get_sql_type re-exported DATA_DB, INDEXED_FORMATS, + LOAD_TIMEOUT_SECONDS, NewerFileSaved, build_data_table, data_table_row_exists, @@ -69,7 +70,9 @@ } -def index_resource_data(resource: Resource) -> Optional[ResourceDataTable]: +def index_resource_data( + resource: Resource, load_timeout: Optional[int] = LOAD_TIMEOUT_SECONDS +) -> Optional[ResourceDataTable]: """Copy a resource's tabular file into a table of its own in the data database. Every upload gets a new table, named from its new ResourceDataTable row. @@ -80,8 +83,9 @@ def index_resource_data(resource: Resource) -> Optional[ResourceDataTable]: data stays as it was and this build's table is dropped. Must be called outside any transaction, so "committed" really means - committed. Returns the new ResourceDataTable, or None when the resource - was not switched over. + committed. ``load_timeout`` limits reading the file (seconds; None for no + limit, as the migration command uses). Returns the new ResourceDataTable, + or None when the resource was not switched over. """ resource_id = str(resource.id) @@ -95,7 +99,7 @@ def index_resource_data(resource: Resource) -> Optional[ResourceDataTable]: return None # Read the file into a frame, remembering exactly which stored file it was. - loaded = load_resource_frame(resource) + loaded = load_resource_frame(resource, timeout=load_timeout) if loaded is None: return None diff --git a/api/utils/data_table_migration.py b/api/utils/data_table_migration.py new file mode 100644 index 00000000..0000e231 --- /dev/null +++ b/api/utils/data_table_migration.py @@ -0,0 +1,475 @@ +"""Helpers for ``migrate_data_table_names``: moving old data tables to per-upload names. + +Old tables were named from the resource name and dataset title. Postgres cut +those names to 63 bytes, so two resources could end up sharing one table. The +command works out what each ResourceDataTable row needs, then: + + - a short old name (63 bytes or less) can't have collided, so its table is + renamed in place to "rdt_"; + - a long old name may already hold another resource's data, so the + resource is rebuilt from its own stored file (through the normal upload + flow, with no limit on reading time); the shared table is dropped by the + delete receiver once its last row has moved. If the file is gone or can't + become a table, a table only this row uses is renamed and marked + "unverified" in its comment (listed on every run until the resource is + re-uploaded); a shared one is left for a manual fix; + - a row whose table was renamed but not yet pointed at (a crashed run) just + gets its row updated; + - a row with no table at all is re-indexed from its file; + - tables no row uses are listed as orphans, and dropped only from a report + someone reviewed. + +A dry run checks each stored file too, so its report shows what --apply will +do. Each step is safe to rerun; every row's outcome is logged as it happens. +Logs carry ids, outcomes and counts only. +Docs: ./data_indexing_architecture.md +""" + +import json +import os +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple + +import structlog +from django.core.management.base import CommandError +from django.db import connections, transaction +from django.db.utils import OperationalError +from psycopg2 import sql as pg_sql # type: ignore[import-untyped] + +from api.models import AccessModelResource, Resource, ResourceDataTable +from api.utils.data_indexing import index_resource_data +from api.utils.data_table_names import ( + DATA_TABLE_PREFIX, + MAX_IDENTIFIER_BYTES, + data_table_name, + physical_table_name, +) +from api.utils.data_tables import ( + DATA_DB, + DataDbTable, + describe_data_table, + django_table_names, + drop_data_table, + error_fields, + has_indexable_file, + lock_resource, + table_built_at, + table_is_unverified, +) + +logger = structlog.get_logger("dataspace.data_table_migration") + +# How long a rename waits for readers holding the table before it is skipped. +RENAME_LOCK_TIMEOUT = "2s" +# Per-upload tables younger than this may belong to an upload that hasn't switched yet. +YOUNG_TABLE_AGE = timedelta(hours=1) +LOCK_NOT_AVAILABLE = "55P03" +# Outcomes that leave nothing for a rerun to do (missing or unusable files need a person). +NEEDS_MANUAL_FIX = { + "missing_file", + "not_indexable", + "planned_missing_file", + "planned_not_indexable", +} +SETTLED_OUTCOMES = NEEDS_MANUAL_FIX | {"already_migrated"} + + +class DataTableRow(NamedTuple): + """The fields of a ResourceDataTable row the migration looks at.""" + + row_id: Any + resource_id: Any + table_name: str + created: datetime + + +@dataclass +class MigrationPlan: + """What each row needs. Rebuilds carry whether another row shares the table.""" + + already_migrated: List[DataTableRow] = field(default_factory=list) + row_updates: List[DataTableRow] = field(default_factory=list) + renames: List[DataTableRow] = field(default_factory=list) + rebuilds: List[Tuple[DataTableRow, bool]] = field(default_factory=list) + missing_tables: List[DataTableRow] = field(default_factory=list) + orphans: List[DataDbTable] = field(default_factory=list) + shared_groups: int = 0 # old tables two or more rows point at + + +def read_data_table_rows() -> List[DataTableRow]: + """Every ResourceDataTable row, oldest first (one row per tabular resource).""" + rows = ( + ResourceDataTable.objects.order_by("created") + .values_list("id", "resource_id", "table_name", "created") + .iterator() + ) + return [DataTableRow(*row) for row in rows] + + +def plan_table_migration( + rows: List[DataTableRow], + tables: List[DataDbTable], + now: datetime, + resource_id: Optional[str] = None, +) -> MigrationPlan: + """Sort rows by what they need, given the tables that exist now. + + ``resource_id`` limits the actions to one resource; orphans are always + worked out against every row. + """ + existing = {table.name for table in tables} + rows_per_table = Counter(physical_table_name(row.table_name) for row in rows) + plan = MigrationPlan( + orphans=find_orphans(rows, tables, now), + shared_groups=sum(1 for count in rows_per_table.values() if count > 1), + ) + for row in rows: + if resource_id and str(row.resource_id) != str(resource_id): + continue + own_name = data_table_name(row.row_id) + physical = physical_table_name(row.table_name) + if row.table_name == own_name: + target = plan.already_migrated if own_name in existing else plan.missing_tables + target.append(row) + elif physical in existing and _is_long(row.table_name): + plan.rebuilds.append((row, rows_per_table[physical] > 1)) + elif physical in existing: + plan.renames.append(row) + elif own_name in existing: + plan.row_updates.append(row) + else: + plan.missing_tables.append(row) + return plan + + +def _is_long(name: str) -> bool: + return len(name.encode("utf-8")) > MAX_IDENTIFIER_BYTES + + +def find_orphans( + rows: List[DataTableRow], tables: List[DataDbTable], now: datetime +) -> List[DataDbTable]: + """Tables no row uses, in the order given. + + Never a Django model's table, never a row's (cut) table or its renamed + "rdt_" table, and never a per-upload table built less than + YOUNG_TABLE_AGE ago (its upload may still be switching over). + """ + in_use = {physical_table_name(row.table_name) for row in rows} + in_use |= {data_table_name(row.row_id) for row in rows} + protected = in_use | django_table_names() + orphans = [] + for table in tables: + if table.name in protected: + continue + built_at = table_built_at(table.comment) + if ( + table.name.startswith(DATA_TABLE_PREFIX) + and built_at + and now - built_at < YOUNG_TABLE_AGE + ): + continue + orphans.append(table) + return orphans + + +def point_row_at_its_table(row: DataTableRow) -> str: + """Finish a rename a crashed run started: point the row at "rdt_".""" + try: + with transaction.atomic(): + lock_resource(Resource.objects.get(pk=row.resource_id)) + updated = ResourceDataTable.objects.filter( + pk=row.row_id, table_name=row.table_name + ).update(table_name=data_table_name(row.row_id)) + except Exception as error: + logger.error("data_table_row_update_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + return "row_updated" if updated else "changed" + + +def rename_legacy_table(row: DataTableRow, unverified: bool = False) -> str: + """Rename the row's old table to "rdt_" and point the row at it. + + ``unverified`` stores the "unverified" mark in the table's comment, for a + long-named table kept without a rebuild (it may hold another resource's data). + + Holds the resource row, so an upload can't switch it meanwhile; a row an + upload already replaced is left alone ("changed"). Waits at most + RENAME_LOCK_TIMEOUT for readers ("skipped_lock"). The data database commits + first; if the row update then fails, a rerun finds the renamed table and + updates the row ("row_updated"). Returns "renamed" on success, "failed" + on any other error (logged). + """ + new_name = data_table_name(row.row_id) + try: + with transaction.atomic(): + resource = Resource.objects.select_related("dataset").get(pk=row.resource_id) + lock_resource(resource) + still_legacy = ResourceDataTable.objects.filter( + pk=row.row_id, table_name=row.table_name + ).exists() + if not still_legacy: + return "changed" + comment = describe_data_table(resource, built_at=row.created, unverified=unverified) + _rename_in_data_db(row.table_name, new_name, comment) + ResourceDataTable.objects.filter(pk=row.row_id).update(table_name=new_name) + except OperationalError as error: + if error_fields(error)["error_code"] == LOCK_NOT_AVAILABLE: + logger.warning("data_table_rename_skipped_lock", row_id=str(row.row_id)) + return "skipped_lock" + logger.error("data_table_rename_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + except Exception as error: + logger.error("data_table_rename_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + logger.info("data_table_renamed", row_id=str(row.row_id), table_name=new_name) + return "renamed" + + +def _rename_in_data_db(old_name: str, new_name: str, comment: str) -> None: + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL lock_timeout = {}").format(pg_sql.Literal(RENAME_LOCK_TIMEOUT)) + ) + cursor.execute( + pg_sql.SQL("ALTER TABLE {} RENAME TO {}").format( + pg_sql.Identifier(old_name), pg_sql.Identifier(new_name) + ) + ) + cursor.execute( + pg_sql.SQL("COMMENT ON TABLE {} IS {}").format( + pg_sql.Identifier(new_name), pg_sql.Literal(comment) + ) + ) + + +def file_problem(resource: Resource) -> Optional[str]: + """Why the resource's stored file can't become a table, or None if it can. + + "missing_file" (no file, or gone from storage) or "not_indexable" (a format + that never gets a table, e.g. the file was later replaced by a PDF). + Storage errors are raised. + """ + details = getattr(resource, "resourcefiledetails", None) + if not details or not details.file: + return "missing_file" + if not has_indexable_file(resource): + return "not_indexable" + if not details.file.storage.exists(details.file.name): + return "missing_file" + return None + + +def rebuild_decision(row: DataTableRow, shared: bool) -> str: + """What --apply will do with a long-named old table. + + "rebuild" when the stored file can become a table; otherwise + "unverified_rename" for a table only this row uses, or the file problem + ("missing_file" / "not_indexable") for a shared one, which is left alone. + "check_failed" (logged) when the check itself fails. + """ + try: + problem = file_problem(Resource.objects.get(pk=row.resource_id)) + except Exception as error: + logger.error("data_table_file_check_failed", row_id=str(row.row_id), **error_fields(error)) + return "check_failed" + if problem is None: + return "rebuild" + return problem if shared else "unverified_rename" + + +def reindex_decision(row: DataTableRow) -> str: + """What --apply will do with a row whose table is gone: "reindex" or the file problem.""" + try: + problem = file_problem(Resource.objects.get(pk=row.resource_id)) + except Exception as error: + logger.error("data_table_file_check_failed", row_id=str(row.row_id), **error_fields(error)) + return "check_failed" + return problem or "reindex" + + +def rebuild_from_file(row: DataTableRow, shared: bool) -> str: + """Carry out the rebuild decision for a long-named old table. + + Rebuilds go through the normal upload flow with no limit on reading time, + so the resource gets a fresh per-upload table and the old one is dropped + once no row uses it. Returns "rebuilt", "unverified" (renamed and marked), + the rename's other outcomes, a file problem, "check_failed" or "failed". + """ + decision = rebuild_decision(row, shared) + if decision == "unverified_rename": + outcome = rename_legacy_table(row, unverified=True) + return "unverified" if outcome == "renamed" else outcome + if decision != "rebuild": + return decision + return _index_without_time_limit(row) + + +def reindex_missing_table(row: DataTableRow) -> str: + """Re-index a row whose table is gone: "reindexed", a file problem, or a failure.""" + decision = reindex_decision(row) + if decision != "reindex": + return decision + outcome = _index_without_time_limit(row) + return "reindexed" if outcome == "rebuilt" else outcome + + +def _index_without_time_limit(row: DataTableRow) -> str: + try: + resource = Resource.objects.select_related("dataset").get(pk=row.resource_id) + return "rebuilt" if index_resource_data(resource, load_timeout=None) else "failed" + except Exception as error: + logger.error("data_table_rebuild_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + + +def find_unverified(rows: List[DataTableRow], tables: List[DataDbTable]) -> List[DataTableRow]: + """Rows whose table carries the "unverified" mark — kept without a rebuild, needing a check.""" + marked = {table.name for table in tables if table_is_unverified(table.comment)} + return [ + row + for row in rows + if physical_table_name(row.table_name) in marked or data_table_name(row.row_id) in marked + ] + + +def access_models_touched(resource_ids: Iterable[Any]) -> List[str]: + """Access models whose column choices a rebuild resets (columns are recreated).""" + ids = ( + AccessModelResource.objects.filter(resource_id__in=list(resource_ids)) + .values_list("access_model_id", flat=True) + .distinct() + ) + return sorted(str(access_model_id) for access_model_id in ids) + + +def read_reviewed_orphans(path: str) -> Set[str]: + """The orphan table names listed in a report someone reviewed. + + Raises CommandError if the file can't be read or isn't a report, so a bad + path stops the run before it changes anything. + """ + try: + with open(path, encoding="utf-8") as report: + return {orphan["name"] for orphan in json.load(report)["orphans"]} + except (OSError, ValueError, KeyError, TypeError) as error: + raise CommandError(f"Can't read the reviewed report {path}: {error}") from error + + +def same_file(first: str, second: str) -> bool: + """Whether two paths name the same file.""" + return os.path.abspath(first) == os.path.abspath(second) + + +def drop_reviewed_orphans(reviewed: Set[str], current_orphans: List[DataDbTable]) -> List[str]: + """Drop tables that are both in the reviewed report and still orphans now.""" + dropped = [] + for table in current_orphans: + if table.name in reviewed and drop_data_table(table.name): + dropped.append(table.name) + return dropped + + +def write_report(path: str, report: Dict[str, Any]) -> None: + """Write the run's report as JSON (ids, names, outcomes and sizes; no data).""" + with open(path, "w", encoding="utf-8") as out: + json.dump(report, out, indent=2, default=str) + + +def record_outcome( + row: DataTableRow, action: str, outcome: str, shared: Optional[bool] = None +) -> Dict[str, Any]: + """Log one row's outcome as it happens, and return its line for the report.""" + entry: Dict[str, Any] = { + "row_id": str(row.row_id), + "resource_id": str(row.resource_id), + "table_name": row.table_name, + "action": action, + "outcome": outcome, + } + if shared is not None: + entry["shared"] = shared + logger.info( + "data_table_migration_row", + row_id=entry["row_id"], + resource_id=entry["resource_id"], + action=action, + outcome=outcome, + ) + return entry + + +def nothing_left_to_do(entries: List[Dict[str, Any]]) -> bool: + """Whether a rerun would change nothing: every row is migrated or needs a person.""" + return all(entry["outcome"] in SETTLED_OUTCOMES for entry in entries) + + +def build_report( + mode: str, + generated_at: datetime, + entries: Optional[List[Dict[str, Any]]] = None, + orphans: Optional[List[DataDbTable]] = None, + orphans_dropped: Optional[List[str]] = None, + access_models: Optional[List[str]] = None, + unverified: Optional[List[DataTableRow]] = None, + shared_groups: int = 0, + status: str = "complete", +) -> Dict[str, Any]: + """The run's JSON report: counts by outcome, per-row lines, orphans, unverified tables. + + Orphans carry their size and comment (resource, dataset, build time) so a + reviewer can tell what each is before approving its drop. + """ + entries = entries or [] + counts = Counter(entry["outcome"] for entry in entries) + return { + "mode": mode, + "status": status, + "generated_at": generated_at.isoformat(), + "counts": dict(sorted(counts.items())), + "shared_groups": shared_groups, + "rows": entries, + "needs_manual_fix": [e for e in entries if e["outcome"] in NEEDS_MANUAL_FIX], + "unverified_tables": [ + {"row_id": str(r.row_id), "resource_id": str(r.resource_id), "table_name": r.table_name} + for r in unverified or [] + ], + "orphans": [ + {"name": t.name, "size_bytes": t.size_bytes, "comment": t.comment} + for t in orphans or [] + ], + "orphans_dropped": orphans_dropped or [], + "access_models_touched": access_models or [], + } + + +def summary_lines(report: Dict[str, Any], report_path: str) -> List[str]: + """A short plain-text summary of a report, for the terminal and the logs.""" + orphan_bytes = sum(orphan["size_bytes"] for orphan in report["orphans"]) + lines = [f"{report['mode']}: report written to {report_path}"] + lines += [f" {outcome}: {count}" for outcome, count in report["counts"].items()] + lines.append(f" old tables shared by several rows: {report['shared_groups']}") + lines.append( + f" rows needing a manual fix (file missing or unusable): {len(report['needs_manual_fix'])}" + ) + lines.append( + f" unverified tables (may hold another resource's data): {len(report['unverified_tables'])}" + ) + lines.append(f" orphans listed: {len(report['orphans'])} ({orphan_bytes} bytes)") + lines.append(f" orphans dropped: {len(report['orphans_dropped'])}") + lines.append( + f" access models with column choices reset: {len(report['access_models_touched'])}" + ) + logger.info( + "data_table_migration_finished", + mode=report["mode"], + shared_groups=report["shared_groups"], + needs_manual_fix=len(report["needs_manual_fix"]), + unverified_tables=len(report["unverified_tables"]), + orphans_listed=len(report["orphans"]), + orphans_dropped=len(report["orphans_dropped"]), + **report["counts"], + ) + return lines diff --git a/api/utils/data_tables.py b/api/utils/data_tables.py index de4603bc..776967f0 100644 --- a/api/utils/data_tables.py +++ b/api/utils/data_tables.py @@ -5,6 +5,8 @@ ResourceDataTable row over to that table in one main-database commit. Old tables are dropped after the commit by the row's delete receiver (api/signals/data_table_signals.py). Each helper here does one step of that. +The last few (listing the data database's tables, the "is this really a +separate database?" check) serve the ``migrate_data_table_names`` command. Tables live in the separate ``data_db`` database (its statements autocommit); the rows that point at them (ResourceDataTable, ResourceSchema) live in the @@ -23,10 +25,11 @@ from collections import Counter from datetime import datetime from io import StringIO -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional, Set import pandas as pd import structlog +from django.apps import apps from django.db import connections, transaction from psycopg2 import sql as pg_sql # type: ignore[import-untyped] @@ -130,21 +133,23 @@ def has_indexable_file(resource: Resource) -> bool: return True -def load_resource_frame(resource: Resource) -> Optional[LoadedFile]: +def load_resource_frame( + resource: Resource, timeout: Optional[int] = LOAD_TIMEOUT_SECONDS +) -> Optional[LoadedFile]: """Read the resource's stored file into a frame and note which file it was. Column names are cleaned the way tables have always needed them: double quotes removed, line breaks and tabs turned into spaces, and names that then collide get "_1", "_2", ... suffixes. - Returns None when the file can't be read, takes longer than - LOAD_TIMEOUT_SECONDS, or has no rows (each case logged). + Returns None when the file can't be read, takes longer than ``timeout`` + seconds (None: no limit), or has no rows (each case logged). """ file_details = resource.resourcefiledetails identity = FileIdentity(file_details.file.name, file_details.sha256) try: with local_copy(file_details.file) as path: - frame = _load_with_timeout(path, file_details.format.lower()) + frame = _load_with_timeout(path, file_details.format.lower(), timeout) except Exception as error: logger.error("data_index_load_failed", resource_id=str(resource.id), **error_fields(error)) return None @@ -158,17 +163,20 @@ def load_resource_frame(resource: Resource) -> Optional[LoadedFile]: return LoadedFile(frame, identity) -def _load_with_timeout(path: str, file_format: str) -> Optional[pd.DataFrame]: +def _load_with_timeout( + path: str, file_format: str, timeout: Optional[int] +) -> Optional[pd.DataFrame]: # SIGALRM only works in a Unix process's main thread; elsewhere (a server # worker thread) the file loads without a limit, as it always has. - if threading.current_thread() is not threading.main_thread() or not hasattr(signal, "SIGALRM"): + in_main_thread = threading.current_thread() is threading.main_thread() + if timeout is None or not in_main_thread or not hasattr(signal, "SIGALRM"): return load_tabular_data(path, file_format) def on_alarm(signum: int, frame: Any) -> None: - raise TimeoutError(f"loading took longer than {LOAD_TIMEOUT_SECONDS} seconds") + raise TimeoutError(f"loading took longer than {timeout} seconds") previous = signal.signal(signal.SIGALRM, on_alarm) - signal.alarm(LOAD_TIMEOUT_SECONDS) + signal.alarm(timeout) try: return load_tabular_data(path, file_format) finally: @@ -203,21 +211,24 @@ def new_data_table_row(resource: Resource) -> ResourceDataTable: return row -def describe_data_table(resource: Resource, built_at: datetime) -> str: +def describe_data_table(resource: Resource, built_at: datetime, unverified: bool = False) -> str: """The comment stored on a new table, so people browsing the data database know what it holds. JSON with the resource id and name, the dataset title and the build time; the migration command reads ``built_at`` to leave freshly built tables alone. + ``unverified`` marks an old table the migration kept without rebuilding it + (its file was missing), which may hold another resource's data; the mark + stays until the resource's next upload replaces the table. """ - return json.dumps( - { - "resource_id": str(resource.id), - "resource": resource.name, - "dataset": resource.dataset.title, - "built_at": built_at.isoformat(), - }, - ensure_ascii=False, - ) + description: Dict[str, Any] = { + "resource_id": str(resource.id), + "resource": resource.name, + "dataset": resource.dataset.title, + "built_at": built_at.isoformat(), + } + if unverified: + description["unverified"] = True + return json.dumps(description, ensure_ascii=False) def build_data_table(table_name: str, frame: pd.DataFrame, comment: str) -> None: @@ -376,3 +387,67 @@ def _another_row_uses(table_name: str, row_id: Any) -> bool: .values_list("table_name", flat=True) ) return any(physical_table_name(name) == physical for name in candidates) + + +class DataDbTable(NamedTuple): + """A table in the data database's public schema.""" + + name: str + size_bytes: int + comment: Optional[str] + + +class DataDbNotSeparate(Exception): + """The data database looks like (or holds tables of) the main database.""" + + +def list_data_db_tables() -> List[DataDbTable]: + """Every ordinary table in the data database's public schema, with its size and comment.""" + with connections[DATA_DB].cursor() as cursor: + cursor.execute( + "SELECT c.relname, pg_total_relation_size(c.oid), obj_description(c.oid, 'pg_class') " + "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'public' AND c.relkind = 'r' ORDER BY c.relname" + ) + return [DataDbTable(name, int(size), comment) for name, size, comment in cursor.fetchall()] + + +def django_table_names() -> Set[str]: + """The tables Django's own models use (plus its migration log).""" + models_tables = {m._meta.db_table for m in apps.get_models(include_auto_created=True)} + return models_tables | {"django_migrations"} + + +def assert_data_db_is_separate() -> None: + """Raise DataDbNotSeparate unless the data database is clearly not the main one. + + Refused when both connections name the same host, port and database, or + when the data database holds ``django_migrations`` or any table a Django + model uses — either way, every application table would look like an + unused data table to the orphan sweep. + """ + main = connections["default"].settings_dict + data = connections[DATA_DB].settings_dict + if all( + str(main.get(key) or "") == str(data.get(key) or "") for key in ("HOST", "PORT", "NAME") + ): + raise DataDbNotSeparate("the data database connection points at the main database") + found = sorted(django_table_names() & {table.name for table in list_data_db_tables()}) + if found: + raise DataDbNotSeparate(f"the data database holds Django tables: {', '.join(found[:5])}") + + +def table_built_at(comment: Optional[str]) -> Optional[datetime]: + """When a per-upload table was built, read from its comment (None if unreadable).""" + try: + return datetime.fromisoformat(json.loads(comment or "")["built_at"]) + except (ValueError, KeyError, TypeError): + return None + + +def table_is_unverified(comment: Optional[str]) -> bool: + """Whether a table's comment carries the migration's "unverified" mark.""" + try: + return json.loads(comment or "").get("unverified") is True + except (ValueError, AttributeError): + return False diff --git a/tests/data_index/test_migrate_data_table_names.py b/tests/data_index/test_migrate_data_table_names.py new file mode 100644 index 00000000..03f776f2 --- /dev/null +++ b/tests/data_index/test_migrate_data_table_names.py @@ -0,0 +1,690 @@ +"""The one-off command that moves existing data tables to per-upload names. + +Root cause it cleans up after: tables used to be named from the resource name +and dataset title, cut by Postgres to 63 bytes, so two resources could share +one table (one of them serving the other's data). Short names can't have +collided and are renamed in place; long names may already hold another +resource's data and are rebuilt from each resource's own file. +""" + +import json +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone +from io import StringIO +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pandas as pd +import psycopg2 +import pytest +from django.core.management import CommandError, call_command +from django.db import connections + +from api.models import AccessModel, AccessModelResource, ResourceDataTable +from api.utils.data_table_migration import ( + DataTableRow, + find_orphans, + rename_legacy_table, +) +from api.utils.data_table_names import data_table_name +from api.utils.data_tables import DataDbTable, build_data_table +from tests.data_index.support import uses_data_db + +SHORT_LEGACY = "casescsv_district-health-indicators_1a2b3c4d" +SHARED_PREFIX = "district-wise-prevalence-of-anaemia-among-women-aged-15-49-dist" # 63 bytes +LONG_FIRST = SHARED_PREFIX + "rict-health-indicators-nfhs-4_1a2b3c4d" +LONG_SECOND = SHARED_PREFIX + "rict-health-indicators-nfhs-5_5e6f7a8b" +WRONG_ROWS = [("Somewhere else", 999)] + + +def legacy_table(name: str, rows: List[tuple]) -> None: + """A table as the old code built it (Postgres cuts the name to 63 bytes).""" + with connections["data_db"].cursor() as cursor: + cursor.execute(f'CREATE TABLE "{name}" (district text, cases bigint)') + for row in rows: + cursor.execute(f'INSERT INTO "{name}" VALUES (%s, %s)', list(row)) + + +def legacy_row(resource: Any, name: str) -> ResourceDataTable: + return ResourceDataTable.objects.create(resource=resource, table_name=name) + + +def run_command(*args: str, report: Path) -> str: + out = StringIO() + call_command("migrate_data_table_names", *args, "--report", str(report), stdout=out) + return out.getvalue() + + +def read_report(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text()) + + +def outcomes(report: Dict[str, Any]) -> Dict[str, str]: + return {entry["table_name"]: entry["outcome"] for entry in report["rows"]} + + +def current_table(resource: Any) -> str: + return ResourceDataTable.objects.get(resource=resource).table_name + + +def remove_stored_file(resource: Any) -> None: + details = resource.resourcefiledetails + details.file.storage.delete(details.file.name) + + +# --- Layer 1: the command against both databases ----------------------------- + + +@uses_data_db +def test_a_short_legacy_table_is_renamed_only_on_apply( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10), ("Nagaon", 20)]) + + run_command(report=tmp_path / "dry.json") + assert current_table(resource) == SHORT_LEGACY + assert tables.names() == [SHORT_LEGACY] + assert outcomes(read_report(tmp_path / "dry.json")) == {SHORT_LEGACY: "planned_rename"} + + run_command("--apply", report=tmp_path / "apply.json") + assert current_table(resource) == data_table_name(row.id) + assert tables.names() == [data_table_name(row.id)] + assert tables.rows(data_table_name(row.id)) == [("Kamrup", 10), ("Nagaon", 20)] + assert json.loads(tables.comment(data_table_name(row.id)))["resource_id"] == str(resource.id) + + +@uses_data_db +def test_a_table_two_rows_shared_is_rebuilt_from_each_resources_own_file( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) # the second upload overwrote the first's data + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == { + LONG_FIRST: "rebuilt", + LONG_SECOND: "rebuilt", + } + assert len(tables.rows(current_table(first))) == 3 + assert len(tables.rows(current_table(second))) == 4 + assert sorted(tables.names()) == sorted([current_table(first), current_table(second)]) + + +@uses_data_db +def test_a_long_legacy_table_with_one_row_left_is_rebuilt_not_renamed( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) # its former sharer, since deleted, overwrote it + + run_command("--apply", report=tmp_path / "apply.json") + + assert tables.rows(current_table(resource)) == [("Jorhat", 30), ("Kamrup", 10), ("Nagaon", 20)] + assert tables.names() == [current_table(resource)] + + +@uses_data_db +def test_a_long_legacy_table_whose_file_is_gone_is_renamed_and_marked_unverified( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + row = legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + remove_stored_file(resource) + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == {LONG_FIRST: "unverified"} + assert current_table(resource) == data_table_name(row.id) + assert tables.rows(data_table_name(row.id)) == WRONG_ROWS + assert json.loads(tables.comment(data_table_name(row.id)))["unverified"] is True + + # Every later run still lists it, though the row itself now looks migrated. + run_command(report=tmp_path / "later.json") + later = read_report(tmp_path / "later.json") + assert outcomes(later) == {data_table_name(row.id): "already_migrated"} + assert [t["resource_id"] for t in later["unverified_tables"]] == [str(resource.id)] + + +@uses_data_db +def test_a_sharing_row_whose_file_is_gone_stays_put_while_the_other_moves( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + remove_stored_file(first) + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == { + LONG_FIRST: "missing_file", + LONG_SECOND: "rebuilt", + } + assert current_table(first) == LONG_FIRST + assert SHARED_PREFIX in tables.names() # still used by the first row + assert len(tables.rows(current_table(second))) == 4 + + +@uses_data_db +def test_a_rerun_after_a_crash_between_rename_and_row_update_just_updates_the_row( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + with connections["data_db"].cursor() as cursor: + cursor.execute(f'ALTER TABLE "{SHORT_LEGACY}" RENAME TO "{data_table_name(row.id)}"') + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == {SHORT_LEGACY: "row_updated"} + assert current_table(resource) == data_table_name(row.id) + assert tables.rows(data_table_name(row.id)) == [("Kamrup", 10)] + + +@uses_data_db +def test_a_rename_blocked_by_a_reader_is_skipped_then_finished_on_a_rerun( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + settings = connections["data_db"].settings_dict + reader = psycopg2.connect( + dbname=settings["NAME"], + user=settings["USER"], + password=settings["PASSWORD"], + host=settings["HOST"], + port=settings["PORT"], + ) + try: + with reader.cursor() as cursor: + cursor.execute(f'LOCK TABLE "{SHORT_LEGACY}" IN ACCESS SHARE MODE') + with patch("api.utils.data_table_migration.RENAME_LOCK_TIMEOUT", "200ms"): + run_command("--apply", report=tmp_path / "blocked.json") + finally: + reader.rollback() + reader.close() + + assert outcomes(read_report(tmp_path / "blocked.json")) == {SHORT_LEGACY: "skipped_lock"} + assert current_table(resource) == SHORT_LEGACY + + run_command("--apply", report=tmp_path / "rerun.json") + assert current_table(resource) == data_table_name(row.id) + + +@uses_data_db +def test_orphans_are_listed_with_their_size_and_dropped_only_from_a_reviewed_report( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + owned = add_resource("cases.csv", "districts_v1.csv") + legacy_row(owned, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + legacy_table("forgotten-table_1", [("Kamrup", 1)]) + legacy_table("forgotten-table_2", [("Kamrup", 2)]) + old_build = (datetime.now(dt_timezone.utc) - timedelta(hours=2)).isoformat() + young_build = datetime.now(dt_timezone.utc).isoformat() + frame = pd.DataFrame({"a": [1]}) + build_data_table("rdt_old_orphan", frame, json.dumps({"built_at": old_build})) + build_data_table("rdt_in_flight", frame, json.dumps({"built_at": young_build})) + + run_command(report=tmp_path / "dry.json") + orphans = read_report(tmp_path / "dry.json")["orphans"] + assert sorted(o["name"] for o in orphans) == [ + "forgotten-table_1", + "forgotten-table_2", + "rdt_old_orphan", + ] + assert all(o["size_bytes"] > 0 for o in orphans) + + # Without a reviewed report nothing is dropped. + with pytest.raises(CommandError, match="--from-report"): + run_command("--apply", "--drop-orphans", report=tmp_path / "x.json") + assert "forgotten-table_1" in tables.names() + + # Since the review, a row started using forgotten-table_2, and a new orphan appeared. + late = add_resource("late.csv", "districts_v1.csv") + legacy_row(late, "forgotten-table_2") + legacy_table("not-reviewed", [("Kamrup", 3)]) + + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "dry.json"), + report=tmp_path / "drop.json", + ) + + assert sorted(read_report(tmp_path / "drop.json")["orphans_dropped"]) == [ + "forgotten-table_1", + "rdt_old_orphan", + ] + remaining = tables.names() + assert "forgotten-table_1" not in remaining and "rdt_old_orphan" not in remaining + # The late row's table was renamed by the same run, not dropped. + assert current_table(late).startswith("rdt_") + assert {current_table(late), "not-reviewed", "rdt_in_flight"} <= set(remaining) + + +@uses_data_db +def test_a_second_apply_after_completion_has_nothing_to_do( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + run_command("--apply", report=tmp_path / "first.json") + names_after_first = tables.names() + + output = run_command("--apply", report=tmp_path / "second.json") + + assert "nothing to do" in output + assert tables.names() == names_after_first + assert outcomes(read_report(tmp_path / "second.json")) == { + data_table_name(ResourceDataTable.objects.get(resource=resource).id): "already_migrated" + } + + +@uses_data_db +def test_rebuilds_list_the_access_models_whose_column_choices_reset( + add_resource: Any, dataset: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + access_model = AccessModel.objects.create(name="Partners", dataset=dataset) + AccessModelResource.objects.create(access_model=access_model, resource=resource) + + run_command(report=tmp_path / "dry.json") + + assert read_report(tmp_path / "dry.json")["access_models_touched"] == [str(access_model.id)] + + +@uses_data_db +def test_the_command_refuses_when_the_data_database_is_the_main_database( + tmp_path: Path, +) -> None: + data = connections["data_db"].settings_dict + same = {"HOST": data["HOST"], "PORT": data["PORT"], "NAME": data["NAME"]} + with patch.dict(connections["default"].settings_dict, same): + with pytest.raises(CommandError, match="main database"): + run_command(report=tmp_path / "x.json") + assert not (tmp_path / "x.json").exists() + + +@uses_data_db +def test_the_command_refuses_when_the_data_database_holds_django_tables( + tables: Any, tmp_path: Path +) -> None: + with connections["data_db"].cursor() as cursor: + cursor.execute("CREATE TABLE django_migrations (id int)") + legacy_table("forgotten-table", [("Kamrup", 1)]) + + with pytest.raises(CommandError, match="django_migrations"): + run_command("--apply", report=tmp_path / "x.json") + assert "forgotten-table" in tables.names() + + +@uses_data_db +def test_a_dry_run_shows_what_apply_will_do_with_each_long_table( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + alone = add_resource("alone.csv", "districts_v1.csv") + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + lonely_name = "x" * 70 + legacy_row(alone, lonely_name) + legacy_table(lonely_name, WRONG_ROWS) + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + remove_stored_file(alone) + remove_stored_file(first) + + run_command(report=tmp_path / "dry.json") + + report = read_report(tmp_path / "dry.json") + assert outcomes(report) == { + lonely_name: "planned_unverified_rename", + LONG_FIRST: "planned_missing_file", + LONG_SECOND: "planned_rebuild", + } + assert {e["table_name"]: e["shared"] for e in report["rows"]} == { + lonely_name: False, + LONG_FIRST: True, + LONG_SECOND: True, + } + assert report["shared_groups"] == 1 + assert [e["table_name"] for e in report["needs_manual_fix"]] == [LONG_FIRST] + assert current_table(alone) == lonely_name # a dry run changes nothing + + +@uses_data_db +def test_a_long_table_whose_file_can_no_longer_become_a_table_is_kept_and_marked( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + alone = add_resource("alone.csv", "districts_v1.csv") + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + lonely_name = "x" * 70 + legacy_row(alone, lonely_name) + legacy_table(lonely_name, WRONG_ROWS) + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + for resource in (alone, first): # later replaced by a PDF, which gets no table + resource.resourcefiledetails.format = "pdf" + resource.resourcefiledetails.save() + + run_command("--apply", report=tmp_path / "apply.json") + assert outcomes(read_report(tmp_path / "apply.json")) == { + lonely_name: "unverified", + LONG_FIRST: "not_indexable", # still shared at planning time: left alone + LONG_SECOND: "rebuilt", + } + + # With its sharer gone, the left-behind row is now alone, so a rerun marks it too. + run_command("--apply", report=tmp_path / "rerun.json") + assert outcomes(read_report(tmp_path / "rerun.json"))[LONG_FIRST] == "unverified" + + output = run_command("--apply", report=tmp_path / "last.json") + assert "nothing to do" in output + assert len(read_report(tmp_path / "last.json")["unverified_tables"]) == 2 + + +@uses_data_db +def test_rebuilds_read_files_with_no_time_limit(add_resource: Any, tmp_path: Path) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + + with patch( + "api.utils.data_table_migration.index_resource_data", return_value=MagicMock() + ) as index: + run_command("--apply", report=tmp_path / "apply.json") + + index.assert_called_once() + assert index.call_args.kwargs == {"load_timeout": None} + + +@uses_data_db +def test_a_row_whose_table_is_gone_is_reindexed_or_left_for_a_person( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + has_file = add_resource("has-file.csv", "districts_v1.csv") + no_file = add_resource("no-file.csv", "districts_v1.csv") + gone = ResourceDataTable.objects.create(resource=has_file) # rdt_ name, table never built + ResourceDataTable.objects.create(resource=no_file) + remove_stored_file(no_file) + + output = run_command("--apply", report=tmp_path / "apply.json") + + report = read_report(tmp_path / "apply.json") + assert sorted(e["outcome"] for e in report["rows"]) == ["missing_file", "reindexed"] + assert current_table(has_file) != gone.table_name + assert len(tables.rows(current_table(has_file))) == 3 + assert "nothing to do" not in output + assert "nothing to do" in run_command("--apply", report=tmp_path / "rerun.json") + + +@uses_data_db +def test_resource_id_limits_the_run_to_that_resource( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + target = add_resource("target.csv", "districts_v1.csv") + other = add_resource("other.csv", "districts_v1.csv") + legacy_row(target, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + legacy_row(other, "other-legacy-table") + legacy_table("other-legacy-table", [("Kamrup", 1)]) + + run_command("--apply", "--resource-id", str(target.id), report=tmp_path / "one.json") + + assert current_table(target).startswith("rdt_") + assert current_table(other) == "other-legacy-table" + + +@uses_data_db +def test_a_rename_backs_off_when_an_upload_replaced_the_row_after_planning( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + planned = DataTableRow(row.id, resource.id, SHORT_LEGACY, row.created) + ResourceDataTable.objects.filter(pk=row.pk).update(table_name="rdt_" + "f" * 32) + + assert rename_legacy_table(planned) == "changed" + assert SHORT_LEGACY in tables.names() + + +@uses_data_db +def test_a_bad_reviewed_report_stops_the_run_before_any_change( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + (tmp_path / "edited.json").write_text('{"orphans": [{"name": "a"},]}') # trailing comma + + for reviewed in ("missing.json", "edited.json"): + with pytest.raises(CommandError, match="reviewed report"): + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / reviewed), + report=tmp_path / "out.json", + ) + + assert current_table(resource) == SHORT_LEGACY + + +@uses_data_db +def test_the_report_may_not_overwrite_the_reviewed_report(tmp_path: Path) -> None: + run_command(report=tmp_path / "dry.json") + reviewed = (tmp_path / "dry.json").read_text() + + with pytest.raises(CommandError, match="overwrite"): + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "dry.json"), + report=tmp_path / "dry.json", + ) + + assert (tmp_path / "dry.json").read_text() == reviewed + + +# --- Layer 2: orphan rules --------------------------------------------------- + +NOW = datetime(2026, 11, 13, 12, 0, tzinfo=dt_timezone.utc) + + +def table(name: str, built_minutes_ago: Any = None) -> DataDbTable: + comment = None + if built_minutes_ago is not None: + built_at = (NOW - timedelta(minutes=built_minutes_ago)).isoformat() + comment = json.dumps({"built_at": built_at}) + return DataDbTable(name=name, size_bytes=8192, comment=comment) + + +def test_orphans_never_include_django_tables_rows_tables_or_fresh_builds() -> None: + row_id = __import__("uuid").uuid4() + rows = [ + DataTableRow(row_id=row_id, resource_id=row_id, table_name=LONG_FIRST, created=NOW), + ] + candidates = [ + table("resource_data_table"), # a Django model's table + table("django_migrations"), + table(SHARED_PREFIX), # the row's (cut) table + table(data_table_name(row_id)), # the row's renamed table, row not yet updated + table("rdt_" + "a" * 32, built_minutes_ago=5), # an upload still switching + table("rdt_" + "b" * 32, built_minutes_ago=90), + table("rdt_" + "c" * 32), # no readable build time + table("forgotten-table"), + ] + + names = [t.name for t in find_orphans(rows, candidates, now=NOW)] + + assert names == ["rdt_" + "b" * 32, "rdt_" + "c" * 32, "forgotten-table"] + + +# --- Layer 3: the command's flow (helpers mocked) ---------------------------- + +COMMAND = "api.management.commands.migrate_data_table_names" + + +@pytest.fixture +def command_helpers() -> Any: + parent = MagicMock(unsafe=True) # one helper's name starts with "assert" + names = [ + "assert_data_db_is_separate", + "read_data_table_rows", + "list_data_db_tables", + "plan_table_migration", + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "rebuild_decision", + "reindex_decision", + "reindex_missing_table", + "find_unverified", + "access_models_touched", + "read_reviewed_orphans", + "find_orphans", + "drop_reviewed_orphans", + "write_report", + ] + patchers = [patch(f"{COMMAND}.{name}", getattr(parent, name)) for name in names] + for p in patchers: + p.start() + row = lambda name: DataTableRow( # noqa: E731 + row_id=name, resource_id=name, table_name=name, created=NOW + ) + plan = MagicMock() + plan.already_migrated = [] + plan.row_updates = [row("crashed")] + plan.renames = [row("short")] + plan.rebuilds = [(row("long_old"), False), (row("long_new"), True)] + plan.missing_tables = [row("missing")] + plan.orphans = [] + plan.shared_groups = 1 + parent.plan_table_migration.return_value = plan + parent.point_row_at_its_table.return_value = "row_updated" + parent.rename_legacy_table.return_value = "renamed" + parent.rebuild_from_file.return_value = "rebuilt" + parent.reindex_missing_table.return_value = "reindexed" + parent.rebuild_decision.return_value = "rebuild" + parent.reindex_decision.return_value = "reindex" + parent.find_unverified.return_value = [] + parent.access_models_touched.return_value = [] + parent.drop_reviewed_orphans.return_value = [] + yield parent + for p in patchers: + p.stop() + + +def top_level_calls(parent: MagicMock) -> List[str]: + return [c[0] for c in parent.mock_calls if c[0] and "." not in c[0] and "(" not in c[0]] + + +def test_a_dry_run_checks_safety_first_and_changes_nothing( + command_helpers: MagicMock, tmp_path: Path +) -> None: + call_command( + "migrate_data_table_names", "--report", str(tmp_path / "r.json"), stdout=StringIO() + ) + + calls = top_level_calls(command_helpers) + assert calls[0] == "assert_data_db_is_separate" + assert "write_report" in calls + for action in [ + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "reindex_missing_table", + "drop_reviewed_orphans", + ]: + assert action not in calls + + +def test_apply_fixes_crashed_rows_then_renames_then_rebuilds_in_order( + command_helpers: MagicMock, tmp_path: Path +) -> None: + call_command( + "migrate_data_table_names", + "--apply", + "--report", + str(tmp_path / "r.json"), + stdout=StringIO(), + ) + + actions = [ + (c[0], c.args[0].table_name) + for c in command_helpers.mock_calls + if c[0] + in { + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "reindex_missing_table", + } + ] + assert actions == [ + ("point_row_at_its_table", "crashed"), + ("rename_legacy_table", "short"), + ("rebuild_from_file", "long_old"), + ("rebuild_from_file", "long_new"), + ("reindex_missing_table", "missing"), + ] + command_helpers.drop_reviewed_orphans.assert_not_called() + + +def test_the_command_stops_before_reading_anything_when_safety_fails( + command_helpers: MagicMock, tmp_path: Path +) -> None: + from api.utils.data_tables import DataDbNotSeparate + + command_helpers.assert_data_db_is_separate.side_effect = DataDbNotSeparate("same database") + + with pytest.raises(CommandError): + call_command("migrate_data_table_names", "--apply", "--report", str(tmp_path / "r.json")) + + assert top_level_calls(command_helpers) == ["assert_data_db_is_separate"] + + +def test_orphans_are_dropped_only_with_apply_and_a_reviewed_report( + command_helpers: MagicMock, tmp_path: Path +) -> None: + with pytest.raises(CommandError): + call_command("migrate_data_table_names", "--drop-orphans", "--from-report", "r.json") + command_helpers.drop_reviewed_orphans.assert_not_called() + + command_helpers.read_reviewed_orphans.return_value = {"forgotten-table"} + call_command( + "migrate_data_table_names", + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "reviewed.json"), + "--report", + str(tmp_path / "r.json"), + stdout=StringIO(), + ) + command_helpers.read_reviewed_orphans.assert_called_once_with(str(tmp_path / "reviewed.json")) + command_helpers.drop_reviewed_orphans.assert_called_once_with( + {"forgotten-table"}, command_helpers.find_orphans.return_value + ) diff --git a/tests/journeys/data_index/reupload-and-delete.py b/tests/journeys/data_index/reupload-and-delete.py new file mode 100644 index 00000000..2055b883 --- /dev/null +++ b/tests/journeys/data_index/reupload-and-delete.py @@ -0,0 +1,149 @@ +""" +Journey: upload a CSV, read its rows, replace the file while polling the data +API (it must never come back empty or half-replaced), fire two replacements at +once (the rows served must match the file the download serves), then delete +the resource and get a 404 from the data API. + +On-demand (Layer 5) — runs against a live backend. Reads from env: + KEYCLOAK_TEST_TOKEN a token for a user who can edit TEST_DATASET_ID + TEST_BASE_URL default http://localhost:8000 + TEST_DATASET_ID a draft dataset to add the resource to +Fails loudly on the first assertion that fails. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_DATASET_ID=... TEST_BASE_URL=http://localhost:8000 \ + python tests/journeys/data_index/reupload-and-delete.py + +Afterwards, at least an hour later (the orphan list skips per-upload tables +built within the last hour, as their upload may still be switching), the +operator runs `python manage.py migrate_data_table_names` (a dry run) on the +server and checks the report lists no orphan from the racing uploads. +Docs: api/utils/data_indexing_architecture.md +""" + +import csv +import io +import json +import os +import threading + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +dataset_id = os.environ["TEST_DATASET_ID"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" + +CREATE = """mutation ($ds: UUID!, $file: Upload!) { + createFileResources(fileResourceInput: {dataset: $ds, files: [$file]}) { id } +}""" +REPLACE = """mutation ($id: UUID!, $file: Upload!) { + updateFileResource(fileResourceInput: {id: $id, file: $file}) { __typename } +}""" +DELETE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" + + +def make_csv(label, rows): + """A CSV whose every row carries the label, so a mix of two files is visible.""" + lines = ["district,label,value"] + [f"District {i},{label},{i}" for i in range(rows)] + return ("\n".join(lines) + "\n").encode() + + +def upload(query, variables, file_bytes): + """Post a GraphQL mutation with one file, using the GraphQL multipart spec.""" + operations = {"query": query, "variables": {**variables, "file": None}} + res = requests.post( + graphql, + data={"operations": json.dumps(operations), "map": json.dumps({"0": ["variables.file"]})}, + files={"0": ("journey.csv", file_bytes, "text/csv")}, + headers=headers, + timeout=300, + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +def served_labels(resource_id): + """The labels the data API serves for the resource (all rows), or None on an error.""" + res = requests.get( + f"{base}/api/resources/{resource_id}/data/", + params={"limit": 10000, "columns": "label"}, + headers=headers, + timeout=60, + ) + if res.status_code != 200: + return None + return {row[0] for row in res.json()["rows"]}, res.json()["total"] + + +def downloaded_label(resource_id): + """The label in the file the download endpoint serves.""" + res = requests.get(f"{base}/api/download/resource/{resource_id}", headers=headers, timeout=60) + assert res.status_code == 200, res.text + rows = list(csv.DictReader(io.StringIO(res.content.decode()))) + labels = {row["label"] for row in rows} + assert len(labels) == 1, labels + return labels.pop() + + +# 1. Upload a CSV and read its rows back. +created = upload(CREATE, {"ds": dataset_id}, make_csv("first", 500)) +resource_id = created["createFileResources"][0]["id"] +assert served_labels(resource_id) == ({"first"}, 500) + +# 2. Replace it while polling: every read is the complete old or the complete new file. +seen = [] +done = threading.Event() + + +def poll(): + while not done.is_set(): + seen.append(served_labels(resource_id)) + + +poller = threading.Thread(target=poll) +poller.start() +upload(REPLACE, {"id": resource_id}, make_csv("second", 800)) +done.set() +poller.join() +assert seen, "the poller never read anything" +for reading in seen: + assert reading in (({"first"}, 500), ({"second"}, 800)), reading +assert served_labels(resource_id) == ({"second"}, 800) + +# 3. Two replacements at once: whichever file the download serves, the data API serves too. +race_errors = [] + + +def race(label): + try: + upload(REPLACE, {"id": resource_id}, make_csv(label, 300)) + except Exception as error: # surfaced below: a thread's exception is otherwise lost + race_errors.append(error) + + +racers = [threading.Thread(target=race, args=(label,)) for label in ("third", "fourth")] +for racer in racers: + racer.start() +for racer in racers: + racer.join() +assert not race_errors, race_errors +winner = downloaded_label(resource_id) +assert served_labels(resource_id) == ({winner}, 300), (winner, served_labels(resource_id)) + +# 4. Delete the resource: its data is gone. +res = requests.post( + graphql, json={"query": DELETE, "variables": {"id": resource_id}}, headers=headers, timeout=60 +) +assert res.status_code == 200 and not res.json().get("errors"), res.text +gone = requests.get(f"{base}/api/resources/{resource_id}/data/", headers=headers, timeout=60) +assert gone.status_code == 404, gone.text + +print("PASS: reupload-and-delete") +print( + "In an hour, run `python manage.py migrate_data_table_names` and check the report " + "lists no orphans (younger per-upload tables are skipped by design)." +) From 599b8482a9dd52c588c1726eaf03b3a47cb334fe Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 06:04:41 +0530 Subject: [PATCH 50/55] docs(data-index): add the data index architecture doc and runbook Covers per-upload table naming, the build -> switch -> drop-after-commit flow, the delete receiver and its shared-table guard, the readers, the migrate_data_table_names command with its runbook and old naming schemes, the Postgres test setup, the failure model, and known limitations. SDK: no. The data API doc links to it. Plan: plans/claude/data-index-table-names.md (Phase 4) --- api/utils/data_indexing_architecture.md | 339 ++++++++++++++++++++++++ docs/dataset_data_api.md | 3 +- 2 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 api/utils/data_indexing_architecture.md diff --git a/api/utils/data_indexing_architecture.md b/api/utils/data_indexing_architecture.md new file mode 100644 index 00000000..0ca820a3 --- /dev/null +++ b/api/utils/data_indexing_architecture.md @@ -0,0 +1,339 @@ +# Data index — per-upload data tables + +> Backend only. Related: [`docs/dataset_data_api.md`](../../docs/dataset_data_api.md) (the public data API that reads these tables). Plan: `plans/claude/data-index-table-names.md`. + +## Overview + +Every tabular resource file (csv, xls, xlsx, ods, parquet, feather, json, tsv) is copied into a table of its own in the separate `data_db` Postgres database. Previews, row counts, the column list, the data API (`/api/resources//data/`, `/api/datasets//data/`) and chart queries read those tables. A resource's `ResourceDataTable` row (main database) records which table is its own; every reader looks the table up from that row, never from a name rule. + +This feature makes each upload's table **unique and safely replaced**: + +- **Names come from the upload's own row id:** `rdt_<32 hex of the ResourceDataTable id>`, 36 characters. They never collide and never change when a resource or dataset is renamed. +- **Re-uploads build first, then switch, then drop.** The new table is built completely, one main-database commit points the resource at it, and the old table is dropped only after that commit. Readers see the complete old data, then the complete new data. A failed re-upload leaves the old data exactly as it was. +- **Deletes free storage.** Deleting a resource, a dataset (by cascade), or a row in the admin drops the table once the delete commits. A rolled-back delete keeps it. +- **Existing tables are migrated by a one-off command** (`migrate_data_table_names`). Its runbook is below. + +Why it was needed: names used to be `slug(resource name)[:50]_slug(dataset title)[:50]_<8 hex>`, up to 110 characters. Postgres keeps only the first 63 bytes of any name, so the id suffix was usually cut off and two resources with long, similar names shared one physical table. A re-upload of one replaced the other's data, and the other's preview, charts and data API served the wrong numbers. Re-uploads also dropped the old table before building the new one: readers saw empty or half-written data for a few seconds, a failed re-upload lost the data for good, and deletes never dropped anything. + +## Submodule map + +| Submodule | Trigger | Code | +|---|---|---| +| Upload / re-index flow | GraphQL `createFileResources`, `updateFileResource` (with a file), `resetFileResourceSchema`, `updateFileResourceSchema` (when a column format changes); `python manage.py update_dataindex` | `index_resource_data` in `api/utils/data_indexing.py`; helpers in `api/utils/data_tables.py` | +| Table names | used by the model and the flow | `api/utils/data_table_names.py` | +| Delete clean-up | `post_delete` of any `ResourceDataTable` row (re-upload, resource/dataset delete, admin delete) | `api/signals/data_table_signals.py` → `drop_data_table_if_unshared` | +| Readers | GraphQL resource fields (preview, row count), data API views, chart generator | `get_row_count`, `get_preview_data`, `get_resource_columns`, `fetch_resource_data`, `query_resource_data` in `api/utils/data_indexing.py` (unchanged) | +| Migration of existing tables | `python manage.py migrate_data_table_names` (one-off, operator-run) | `api/management/commands/migrate_data_table_names.py`; helpers in `api/utils/data_table_migration.py` | + +--- + +## Submodule: Upload / re-index flow + +### Trigger +`index_resource_data(resource, load_timeout=60)` is called synchronously by the resource file mutations in `api/schema/resource_schema.py` (none of them runs inside a transaction), by `update_dataindex` per resource, and by the migration command for rebuilds (with `load_timeout=None`). + +### Business use case +A publisher uploads or replaces a tabular file. Visitors then preview it, chart it and query it through the data API. They must never see another resource's data, an empty table mid-replace, or lose the old data because a new file was bad. + +### Flow (English) +1. **Refuse inside an outer transaction.** If the main database is already in an atomic block, log `data_index_refused_inside_transaction` and return None. Inside someone else's transaction, "committed" would only mean a savepoint was released, and a later rollback would leave a switched table pointing nowhere. +2. **Check the file.** The resource must have a stored file in an indexed format; otherwise return None. Nothing is built. +3. **Load the file** into a frame, and note which stored file it was (storage name + `sha256`). Loading is limited to 60 s where the platform allows it: SIGALRM works only in a process's main thread, so management commands get the limit and web worker threads never have. Column names are cleaned: double quotes removed, line breaks and tabs become spaces, and names that then collide get `_1`, `_2`… An unreadable file, a timeout or an empty frame returns None. +4. **Make this upload's row in memory.** A new `ResourceDataTable` with its UUID, so its table name `rdt_` is known before anything is saved. +5. **Build the new table** in one `data_db` transaction: create it, copy the rows in through a temporary table (`ON COMMIT DROP`), and store a JSON comment (resource id and name, dataset title, `built_at`). A failure, such as a value out of range mid-copy, rolls the whole table back, and the old row and table are untouched. +6. **Switch, in one main-database transaction:** + 1. lock the resource row (`select_for_update`), so two uploads of one resource switch one after the other; + 2. re-read the stored file's identity, and if a newer file was saved while this one was building, back off (`NewerFileSaved`), because the newest file wins; + 3. delete the old `ResourceDataTable` row, if any. Its delete receiver schedules the old table's drop for after the commit; + 4. save the new row, carrying over the old row's `created` (see "Dataset default table" below); + 5. rebuild the `ResourceSchema` column rows (descriptions people wrote are kept by column name). Errors escape, and a header longer than the 255-character `field_name` is refused up front. +7. **Confirm the switch.** It counts only if the new row exists after the transaction. +8. **`finally`: drop this build's table unless step 7 confirmed the switch.** That covers a failed build, a failed switch, a newer file, a missing row, and even `KeyboardInterrupt`. +9. After the commit, the receiver drops the old table (see Delete clean-up). + +Returns the new `ResourceDataTable`, or None when the resource was not switched (each reason logged). + +**Why the switch lets errors escape.** The old column rebuild caught each column's error. Inside an atomic block a database error marks the whole block for rollback, so one bad header silently rolled the switch back while the upload still looked successful. + +**Why the newest file wins.** Each upload request saves its own file before indexing it. Without the re-check, a slow build of an older file could switch in last: the download would serve one file while previews, charts and the data API served another. + +**Dataset default table.** `GET /api/datasets//data/` without `resource_id` serves the dataset's data table with the oldest `created`. The new row inherits the old row's `created`, so replacing a file never moves a dataset's default to another resource. The migration doesn't reshuffle defaults either. + +### Helpers (`api/utils/data_tables.py`, Tier 2 unless noted) +- `is_inside_transaction() -> bool`: whether the main DB is in an atomic block. +- `has_indexable_file(resource) -> bool`: has a stored file whose format is in `INDEXED_FORMATS`; logs why not. +- `load_resource_frame(resource, timeout=60) -> LoadedFile | None`: frame plus `FileIdentity(name, sha256)`; None (logged) when unreadable, too slow, or empty. Depends on Tier 1 `local_copy`, `load_tabular_data` (`api/utils/file_utils.py`). +- `stored_file_identity(resource) -> FileIdentity | None`: the current file, read fresh from the database. +- `new_data_table_row(resource) -> ResourceDataTable`: unsaved, named `rdt_`. +- `describe_data_table(resource, built_at, unverified=False) -> str`: the table comment (JSON). `unverified` is used only by the migration. +- `build_data_table(table_name, frame, comment)`: one `data_db` transaction. Raises on failure, leaving nothing behind. Names via `sql.Identifier`, comment via `sql.Literal`. +- `lock_resource(resource)`: `select_for_update` on the resource row (no-op on SQLite). +- `retire_data_table_row(resource) -> datetime | None`: deletes the current row, returns its `created`. +- `save_data_table_row(row, created)`: inserts, then sets `created` with a queryset update (it's `auto_now_add`). +- `rebuild_resource_schema(resource, frame)`: replaces column rows. Raises `ValueError("column N header is longer than 255 characters")` and lets DB errors escape. +- `data_table_row_exists(row_id) -> bool`. +- `drop_data_table(name) -> bool`: `DROP TABLE IF EXISTS` with `SET LOCAL lock_timeout = '5s'`. Never raises: a failure is logged (`data_table_drop_failed`) and the table is left as an orphan for the migration command to report. +- `error_fields(error) -> dict`: `error_type`, Postgres `error_code`, and `where` (innermost frame in `api/`). **Never the message**, because COPY errors quote the offending cell value. +- `get_sql_type(dtype) -> str`: BIGINT / DOUBLE PRECISION / TIMESTAMP / BOOLEAN / TEXT (moved here; still importable from `data_indexing`, as are `DATA_DB` and `INDEXED_FORMATS`). + +`api/utils/data_table_names.py` (no model imports, because the model uses it): +- `data_table_name(row_id) -> "rdt_" + hex`. +- `physical_table_name(name) -> str`: the first 63 bytes, never half a character (what Postgres stores). +- `may_share_physical_table(name) -> bool`: only names of ≥ 60 bytes can be the cut of a longer one. + +### Model rule +`ResourceDataTable.save()` fills a blank `table_name` with `data_table_name(self.id)`. Existing rows keep their stored names. Nothing else generates names. Rolling the code back is safe: old code only generates a name when the field is blank, and every reader uses the stored name. + +--- + +## Submodule: Delete clean-up + +### Trigger +`post_delete` with `sender=ResourceDataTable`. Django sends it for every row in a cascade or a queryset delete, because a receiver is connected (no fast delete). That covers re-upload (`retire_data_table_row`), `deleteFileResource`, `deleteDataset`, admin deletes of a row, resource or dataset, and the migration's rebuilds. + +### Flow (English) +1. The receiver schedules `drop_data_table_if_unshared(table_name, row_id)` with `transaction.on_commit(..., using=, robust=True)`. +2. After the commit, the drop helper checks whether another row still maps to the same physical table. This is possible only for old long names; `rdt_` names skip the check. + 1. If one does, it logs `data_table_drop_skipped_shared` and keeps the table. + 2. Otherwise it drops the table (5 s lock timeout). Any failure is logged and swallowed. + +A rolled-back delete never runs the callback, so its data is kept. + +**Registration.** The module is imported by `api/signals/__init__.py`. A test checks the receiver is connected in a fresh interpreter, because `collaborative_signals.py` once shipped without being imported and never ran. + +**Lock order.** A switch locks the resource, then its data-table row. A resource delete locks the rows, then the resource. A switch and a delete of the same resource at the same moment can therefore deadlock. Postgres aborts one of them, and nothing is lost: the aborted switch drops its own table, and the aborted delete leaves everything in place. + +--- + +## Submodule: Readers (unchanged) +`get_row_count`, `get_preview_data`, `get_resource_columns` (falls back to `information_schema` by table name when there are no column rows), `fetch_resource_data` (data API; column names via `_column_ident`, which doubles `%`), and `query_resource_data` (charts; `{{table}}` replaced by the quoted table name). They all look up `ResourceDataTable` per call and work unchanged on `rdt_` tables. A reader that fetched the old row just before a switch can hit "relation does not exist" once the old table is dropped. It then takes its existing error path (None / 0 / the data API's 500); it never gets the new file's rows under the old row. Readers still compose the table name with f-strings; `rdt_` names are safe in that position. + +--- + +## Submodule: Migration of existing tables — `migrate_data_table_names` + +### Trigger +Operator-run, once after deploy (and safe to rerun any time as an orphan audit). + +### Business use case +Move every existing table to an `rdt_` name and repair tables two resources were sharing. Then list, and after review drop, tables nothing uses (left behind by deletes from before this feature). + +### Flow (English) +1. Check the options. `--drop-orphans` needs `--apply` and `--from-report`, and `--report` must not name the reviewed file. +2. **Safety:** refuse if `data_db` and the main database name the same host, port and database, or if `data_db` holds `django_migrations` or any Django model's table. Either would make every application table look like an orphan. +3. Read the reviewed orphan list now, if dropping. A missing or malformed file stops the run before any change. +4. Write an "in progress" report. This proves the path is writable, and a run that dies leaves a trace. +5. Read every row (oldest first) and every `data_db` table (size and comment), and sort each row (limited to `--resource-id` if given): + - **already migrated**: `rdt_` exists; + - **row update**: a crashed earlier run renamed the table but didn't update the row (the old name is gone, `rdt_` exists); + - **rename**: an old name of ≤ 63 bytes. It can't have collided; + - **rebuild**: an old name of > 63 bytes, whether one row or several point at the physical table. It may already hold another resource's data, so it is never just renamed while its file is usable. Each rebuild carries whether the table is shared; + - **missing table**: neither name exists. +6. **Dry run (the default):** change nothing, but check each rebuild's and missing table's stored file, and record the planned outcome: + - `planned_rebuild`; + - `planned_unverified_rename`: the file is missing or unusable and the table is not shared; + - `planned_missing_file` / `planned_not_indexable`: the table is shared, so it is left for a person; + - `planned_reindex`. +7. **`--apply`**, in this order: + 1. **Row updates.** + 2. **Renames:** in a main-DB transaction, lock the resource and re-check that the row still has its old name (otherwise `changed`). Then, in `data_db`, `SET LOCAL lock_timeout = '2s'`, rename the table and set its comment. Update the row last. A lock timeout gives `skipped_lock`, and a rerun finishes it. + 3. **Rebuilds:** oldest first, through `index_resource_data(resource, load_timeout=None)`. The old row's delete receiver keeps a shared table until its last row has moved. + - If the file is missing or can no longer become a table (for example, later replaced by a PDF) and no other row uses the table, it is renamed and **marked `"unverified": true` in its comment** (`unverified`). + - If the table is shared, it is left (`missing_file` / `not_indexable`). Once its sharers have moved, the next run finds it alone and marks it unverified. + 4. **Missing tables:** re-indexed from the file, or `missing_file` / `not_indexable`. +8. **`--drop-orphans`:** recompute orphans now, and drop only those that are both in the reviewed report and still orphans. +9. Write the full report and print the summary. Every row's outcome is also logged as it happens (`data_table_migration_row`). "nothing to do" is printed when no row needs a rerun: every row is migrated, or needs a person (missing or unusable file). + +**Orphans:** +- **Never orphans:** a Django model's table, any row's physical table or `rdt_`, and an `rdt_` table whose comment says it was built under an hour ago (its upload may still be switching). +- **Listed:** `rdt_` tables without a readable build time, because every table the new code builds has a comment written in the same transaction as its creation. +- Each listed orphan carries its size and comment. + +**Unverified tables** are listed in every report (`unverified_tables`), dry run or apply, until the resource is re-uploaded (the new table has no mark). They may hold another resource's data. Restore the resource's file and re-upload it, or check the data by hand. + +**Report** (JSON, default `data_table_migration_.json` in the working directory): +- `mode`, `status`; +- `counts` by outcome and `shared_groups`; +- `rows` (ids, table name, action, outcome, `shared`); +- `needs_manual_fix`; +- `unverified_tables`; +- `orphans` (name, size, comment), `orphans_dropped`; +- `access_models_touched`: access models whose column choices are reset because the resource's column rows are recreated. This is existing re-upload behaviour. + +No file contents or values appear in the report. + +### Runbook +1. Deploy. New uploads use `rdt_` names from now on, and old names keep working. +2. Off-peak: `python manage.py migrate_data_table_names --report dry.json`. Review `counts`, `shared_groups`, `needs_manual_fix` (restore files first if you can), `unverified_tables` and `orphans`. +3. `python manage.py migrate_data_table_names --apply --report apply1.json`. Rerun until it prints "nothing to do". + - `skipped_lock` and `failed` rows are retried. + - A shared table whose file is missing is marked on the run after its sharers move. +4. At least an hour later: `python manage.py migrate_data_table_names --report orphans.json`. Review the orphan list and remove from the file any table you want kept. +5. `python manage.py migrate_data_table_names --apply --drop-orphans --from-report orphans.json --report drop.json`. +6. Follow up on `unverified_tables` and `needs_manual_fix` by hand. + +Renames take an exclusive lock briefly and wait up to 2 s behind running reads. Rebuilds hold two copies of a table for a moment. + +**Old naming schemes the command meets:** +- `resource_data_<32 hex>`: March–May 2025, 46 characters, so renamed. +- Raw `--<32 hex>`: May–July 2025. These can be mixed case, contain spaces or non-ASCII, and are usually over 63 bytes, so they are rebuilt. +- `slug(name)[:50]_slug(title)[:50]_<8 hex>`: from July 2025. Short ones are renamed and long ones rebuilt. + +Short names of every scheme are unique, so they can't have collided. + +### Helpers (`api/utils/data_table_migration.py`, plus `list_data_db_tables`, `django_table_names`, `assert_data_db_is_separate`, `table_built_at`, `table_is_unverified` in `data_tables.py`) +- `read_data_table_rows()`: every row, oldest first, as `DataTableRow(row_id, resource_id, table_name, created)`. +- `plan_table_migration(rows, tables, now, resource_id=None) -> MigrationPlan`: pure; the sorting above, plus `orphans` and `shared_groups`. +- `find_orphans(rows, tables, now)`: pure; the orphan rules. +- `file_problem(resource)`: `missing_file`, `not_indexable` or None. Storage errors are raised. +- `rebuild_decision(row, shared)` / `reindex_decision(row)`: what `--apply` will do (`check_failed` if the file check errors). +- `point_row_at_its_table`, `rename_legacy_table(row, unverified=False)`, `rebuild_from_file(row, shared)`, `reindex_missing_table(row)`: the actions. Each returns an outcome string and never raises. +- `find_unverified(rows, tables)`: rows whose table carries the mark. +- `access_models_touched(resource_ids)`, `read_reviewed_orphans(path)` (CommandError when unreadable), `same_file`, `drop_reviewed_orphans(reviewed, current)`, `record_outcome` (logs and returns the report line), `nothing_left_to_do`, `build_report`, `write_report`, `summary_lines`. + +--- + +## Data model +- `ResourceDataTable` (`resource_data_table`): `id` UUID pk, `resource` one-to-one, `table_name` CharField(255) unique, `created`, `modified`. **No schema change and no migration file.** Only the generated value changed. `created` is now carried over on re-upload. +- `ResourceSchema`: rebuilt on every re-index (existing behaviour). `AccessModelResource.fields` (many-to-many to these rows) loses its selections whenever they are rebuilt, as before. +- `ResourceFileDetails.file` and `sha256` are read as the file identity (`sha256` is computed on save when the file changes). +- `data_db` tables: one per tabular resource, named `rdt_`, with a JSON comment: `resource_id`, `resource`, `dataset`, `built_at`, and `unverified` when the migration kept a table without a rebuild. +- No Elasticsearch documents are affected. + +## Indexing & performance +- **Switch:** `Resource` by pk (`select_for_update`); `ResourceDataTable` by `resource_id` (one-to-one unique index); `ResourceFileDetails` by `resource_id` (unique). +- **Receiver guard:** `ResourceDataTable.table_name LIKE '<63-byte prefix>%'`, only for old names of ≥ 60 bytes (`rdt_` names skip it). This uses the unique btree on `table_name`, but may be a sequential scan on non-C collations. That's acceptable at one row per tabular resource (thousands). +- **Command:** one ordered read of `ResourceDataTable` (values only, iterator) and one `pg_class` catalogue query on `data_db`. Each is repeated once at the end, and once more when dropping orphans. +- **`data_db`:** no indexes added. Readers do full-table reads with existing timeouts. +- **Disk:** a re-upload briefly holds two copies of the table. +- No new user-facing query, so no pagination change. + +## Security +- No new endpoint and no permission change. The file mutations keep their `ChangeResourcePermission` (edit rights on the dataset) and the data API keeps its existing check. If a user from another organization sends the same mutation, it is refused exactly as before (`tests/schema/test_resource_permissions.py`). +- Out of scope, and unchanged: the public row API serves rows of any PUBLISHED dataset regardless of `access_type`. +- All new SQL composes table names with `sql.Identifier` and comments with `sql.Literal`. Resource and dataset names reach SQL only as a literal comment. +- The orphan sweep refuses to run against a database that looks like the main one, never drops Django tables, and drops only names a person reviewed that are still unused. +- Logs carry ids, outcomes, counts and error codes. Never file contents, cell values or database error messages. + +## SDK impact +SDK: no. This is storage-internal, and no public API changes shape. + +## Tests + +Tests marked `data_db` run against a real Postgres. Locally: + +``` +docker run -d --name dataspace-test-data-db -e POSTGRES_PASSWORD=postgres -p 55432:5432 postgres:16 +export TEST_DATA_DB_URL=postgres://postgres:postgres@localhost:55432/postgres +pytest tests/data_index +``` + +How the setup works: +- **Database:** `tests/test_settings.py` adds the `data_db` alias only when `TEST_DATA_DB_URL` is set. pytest-django creates and drops `test_dataspace_data_db`, and `tests.data_index.routers.DataDbTestRouter` keeps Django models out of it. +- **Skipping:** without the URL these tests are skipped, unless `REQUIRE_DATA_DB_TESTS=1` (set in CI, which runs a `postgres:16` service). Then the run fails, because a run that skipped them proves nothing. +- **Clean-up:** an autouse fixture drops every `public` table after each such test, since pytest-django only empties tables of models routed there, and there are none. +- **Where it applies:** the skip/fail hook and that fixture live in `tests/data_index/conftest.py`, and `uses_data_db` in `tests/data_index/support.py`. Tests elsewhere that need them must move both into a shared conftest. +- **Main database:** stays in-memory SQLite, so row locks are no-ops there. +- **Locally:** if `drf_yasg` fails with "No module named pkg_resources", install `setuptools<81` (CI installs it). + +### Layer 1 — DB helper tests +- **Naming:** a new row is named `rdt_`, and an existing row keeps its stored name on save. Two resources with names identical for 60+ characters get their own tables with their own rows. +- **Uploads:** a first upload creates the row, the table, the column rows and the comment. A re-upload switches to a new table, drops the old one and keeps custom descriptions. +- **Switch timing:** at the moment of the switch the new table is full and the old one still exists, and the lock is taken inside the switch transaction. The old table is dropped only after commit, with no transaction open and the new row saved. +- **Failures keep the old data:** a value out of range mid-copy keeps the old row and table, leaves no new table, and logs error code 22003 without the value. A 300-character header fails the switch, keeps the old row, table and column rows, and drops the new table. +- **Readers and races:** a reader still holding the old name gets "does not exist", never new rows. A slower build of an older file backs off when a newer file was saved, and the newer one stays. +- **Transaction and default:** indexing inside an outer transaction is refused and builds nothing. A re-upload keeps the row's `created` and the dataset's default table in the data API. +- **Deletes:** deleting a resource drops its table; deleting a dataset drops all its tables; a rolled-back delete keeps the table. Two legacy rows sharing a table: the table is kept until the last one is deleted. A drop of a table that's already gone doesn't fail. A drop blocked past the lock timeout is logged and the delete still succeeds. +- **Readers (regression):** on an `rdt_` table with a `%` header, the row count, preview, column list (including the `information_schema` fallback), filtered and ordered fetch, and a chart query all work. +- **Helpers:** indexable-file checks; load records the file identity, returns None for a header-only file, and cleans headers; the column rebuild lets DB errors escape and refuses over-long headers. +- **Migration — moving tables:** + - a short table is renamed only on `--apply`; + - a shared long table is rebuilt from each resource's own file, and the shared table is dropped after both rows move; + - a single long table is rebuilt, not renamed; + - with its file gone, a single long table is renamed and marked unverified, and later runs still list it; + - with its file gone, a shared row stays while the other row moves; + - a run after a crash just updates the row; + - a rename blocked by a reader is skipped, then finished on a rerun. +- **Migration — orphans:** listed with size; drop needs `--from-report`; only reviewed names that are still orphans are dropped; young `rdt_` tables are never listed. +- **Migration — reruns and reports:** + - a second `--apply` has nothing to do; + - access models are listed; + - the dry run shows the planned outcome per long table, the shared flag and `shared_groups`; + - an unusable file (PDF) is marked unverified, or left while shared, and the runs settle on "nothing to do"; + - rebuilds read with no time limit; + - a missing table is re-indexed or left for a person; + - `--resource-id` limits the run; + - a rename backs off when an upload replaced the row after planning. +- **Migration — refusals:** a bad reviewed report stops the run before any change; the report may not overwrite the reviewed file; the run refuses when the data database is the main one or holds `django_migrations`. + +### Layer 2 — Non-DB helper tests +- **Names:** `rdt_` plus lowercase hex, 36 characters. The physical name is cut at 63 bytes and never mid-character. +- **Comment:** a resource name with quotes and semicolons is stored as text, not run as SQL. +- **Orphan rules:** Django tables, a row's tables, and fresh builds are excluded. +- **Registration:** the receiver is connected after app load (fresh interpreter). + +### Layer 3 — Flow tests +- **Upload flow order:** load → name → build → lock → file check → retire → save → rebuild columns → confirm. +- **Clean-up:** this build's table is dropped when the build, lock, retire or column rebuild fails, on a newer file, when the row is missing after commit, and on `KeyboardInterrupt`. It is never dropped after a confirmed switch. +- **Nothing to index:** nothing is built when inside a transaction, when there is no indexable file, or when nothing loaded. +- **Receiver:** schedules one guarded drop per deleted row, via `on_commit(robust=True)`. +- **Command:** the safety check runs first; a dry run changes nothing and writes the report; `--apply` runs row updates → renames → rebuilds (in plan order) → re-indexes; a failed safety check stops before any read; orphans are dropped only with `--apply` and a reviewed report. + +### Layer 4 — Backend API e2e +As the dataset owner, through the GraphQL schema and the data API: +- `createFileResources` gives an `rdt_` table, and the data API returns its rows; +- `updateFileResource` with a new file serves the new rows and the old table is gone; +- `resetFileResourceSchema` switches to a fresh table; +- `deleteFileResource` drops the table, and the data API returns 404; +- `deleteDataset` drops every table. + +Cross-org denial is unchanged and covered by `tests/schema/test_resource_permissions.py`. + +### Layer 5 — API user journey +`tests/journeys/data_index/reupload-and-delete.py`, on request against a running backend (`KEYCLOAK_TEST_TOKEN`, `TEST_DATASET_ID`, `TEST_BASE_URL`): +1. Upload a CSV and read its rows. +2. Replace it while a thread polls the data API. Every read is the complete old file or the complete new one. +3. Fire two replacements at once. The data API serves the same file the download serves. +4. Delete the resource. The data API returns 404. + +At least an hour later, the operator's dry run should list no orphan from the racing uploads. + +### Layer 6 — Browser e2e +n/a — no UI change. + +### Visual checklist (manual, on request) +n/a — no UI change. + +### LLM-judge points +n/a — all assertions are deterministic. + +--- + +## Failure model + +| What could go wrong | Guarantee | +|---|---| +| Two resources map to one physical table | `rdt_` names are unique by construction. | +| A reader sees an empty or half-filled table | The new table is built fully before the switch, and the old one is dropped only after commit. | +| A failed re-upload wipes the data | The old row and table are untouched until the switch commits. | +| A failed build or switch leaves a stray table | `finally` drops it unless the switch is confirmed. | +| A rolled-back delete loses data | Drops run only `on_commit`. | +| The receiver drops a table another row still uses | Shared-name guard for old long names. | +| The receiver isn't loaded | Imported in `api/signals/__init__.py`, and a fresh-interpreter test checks it. | +| Two re-uploads race | Resource row lock plus the newest-file check. | +| A drop fails (lock, missing table) | Logged, never raised; the table becomes an orphan the command reports. | +| A rename gets stuck behind long reads | 2 s lock timeout; skipped, reported, and finished by a rerun. | +| The command crashes between rename and row update | The rerun updates the row; the "unverified" mark survives in the comment. | +| A shared table is rebuilt from the wrong file | Each row is rebuilt from its own file, and the table is dropped only after the last row moves. | +| A stored file is missing or unusable | Reported per row. An unshared table is marked unverified and listed in every report; a shared one is left for a person. | +| A collided table whose sharer was deleted gets renamed | Long old names are always rebuilt while the file is usable. | +| The orphan sweep deletes application tables or an in-flight upload's table | Same-database and Django-table refusals, the one-hour age guard, and only reviewed names dropped. | +| A swallowed error rolls the switch back silently | Column-row errors escape, the switch is confirmed by the row existing, and indexing refuses to run inside an outer transaction. | +| Test data tables leak between tests | An autouse fixture drops every `public` table after each `data_db` test. | + +## Limitations & future work +- **A failed re-upload still reports success to the uploader.** This is existing behaviour: the mutations ignore the indexing result. Now the old data stays served while the download serves the new file, until a successful re-upload. Surfacing the failure would change the upload API's contract and is left for a decision. +- **Two long headers that differ only after 63 bytes** become one column in Postgres, so the build fails. Devanagari headers hit this at about 21 characters. This existed before and is not fixed here. +- **Access-model column selections are reset** whenever column rows are rebuilt (every re-upload, and migration rebuilds). Carrying them over by column name would be a small follow-up. +- **Readers still use f-string table names.** That is safe for `rdt_` names; they were left as they are deliberately. +- The plan's failure table said 5 s for the rename lock timeout. The design and the code use 2 s for renames and 5 s for drops. diff --git a/docs/dataset_data_api.md b/docs/dataset_data_api.md index 5e970ea9..8857b105 100644 --- a/docs/dataset_data_api.md +++ b/docs/dataset_data_api.md @@ -4,7 +4,8 @@ This document describes the HTTP endpoints and SDK methods for fetching the *indexed tabular data* that DataSpace stores in the `data_db` PostgreSQL database. When a CSV / XLSX / Parquet / JSON resource is uploaded, its rows are indexed into a per-resource table so they can be queried, filtered, and -streamed without re-downloading the source file. +streamed without re-downloading the source file. How those tables are built, +named, replaced and cleaned up: `api/utils/data_indexing_architecture.md`. ## Overview From b4bb21f1f103c735491ff6b9b1e329a38f1c04c8 Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 11:00:04 +0530 Subject: [PATCH 51/55] fix(data-index): keep the new table when a run stops after the switch Final QC found that stopping a command (Ctrl+C) while the old table's after-commit drop waited for a reader made the flow's clean-up drop the new table, though the resource already pointed at it. The clean-up now asks the database first: it drops the build's table only if its row isn't saved, and leaves the table for the orphan audit when it can't tell. That also covers a commit that went through but reported an error. The runbook now says a row that fails twice the same way needs a person. Plan: plans/claude/data-index-table-names.md (final QC) --- api/utils/data_indexing.py | 7 ++-- api/utils/data_indexing_architecture.md | 7 +++- api/utils/data_tables.py | 23 ++++++++++ tests/data_index/test_reindex_switch.py | 56 +++++++++++++++++++++---- 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index 6a5af5aa..22159628 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -28,7 +28,7 @@ build_data_table, data_table_row_exists, describe_data_table, - drop_data_table, + drop_unswitched_table, error_fields, get_sql_type, has_indexable_file, @@ -139,9 +139,10 @@ def index_resource_data( except Exception as error: logger.error("data_index_failed", resource_id=resource_id, **error_fields(error)) finally: - # Anything short of a confirmed switch leaves this build's table unused: drop it. + # Short of a confirmed switch, drop this build's table, unless its row was saved + # after all (e.g. a run stopped while the old table's drop was waiting). if not switched: - drop_data_table(new_row.table_name) + drop_unswitched_table(new_row) if not switched: return None diff --git a/api/utils/data_indexing_architecture.md b/api/utils/data_indexing_architecture.md index 0ca820a3..5b82e7d2 100644 --- a/api/utils/data_indexing_architecture.md +++ b/api/utils/data_indexing_architecture.md @@ -48,7 +48,7 @@ A publisher uploads or replaces a tabular file. Visitors then preview it, chart 4. save the new row, carrying over the old row's `created` (see "Dataset default table" below); 5. rebuild the `ResourceSchema` column rows (descriptions people wrote are kept by column name). Errors escape, and a header longer than the 255-character `field_name` is refused up front. 7. **Confirm the switch.** It counts only if the new row exists after the transaction. -8. **`finally`: drop this build's table unless step 7 confirmed the switch.** That covers a failed build, a failed switch, a newer file, a missing row, and even `KeyboardInterrupt`. +8. **`finally`: drop this build's table unless step 7 confirmed the switch.** That covers a failed build, a failed switch, a newer file, a missing row, and even `KeyboardInterrupt`. Before dropping, it checks the database: if the new row was saved after all (a command stopped with Ctrl+C while the old table's after-commit drop waited for a reader, or a commit that went through but reported an error), the table is the resource's live data and stays. If the check itself fails, the table is left for the orphan audit. 9. After the commit, the receiver drops the old table (see Delete clean-up). Returns the new `ResourceDataTable`, or None when the resource was not switched (each reason logged). @@ -72,6 +72,7 @@ Returns the new `ResourceDataTable`, or None when the resource was not switched - `save_data_table_row(row, created)`: inserts, then sets `created` with a queryset update (it's `auto_now_add`). - `rebuild_resource_schema(resource, frame)`: replaces column rows. Raises `ValueError("column N header is longer than 255 characters")` and lets DB errors escape. - `data_table_row_exists(row_id) -> bool`. +- `drop_unswitched_table(row)`: the flow's clean-up. It drops the build's table only if its row isn't saved, and leaves it when the check fails. Never raises. - `drop_data_table(name) -> bool`: `DROP TABLE IF EXISTS` with `SET LOCAL lock_timeout = '5s'`. Never raises: a failure is logged (`data_table_drop_failed`) and the table is left as an orphan for the migration command to report. - `error_fields(error) -> dict`: `error_type`, Postgres `error_code`, and `where` (innermost frame in `api/`). **Never the message**, because COPY errors quote the offending cell value. - `get_sql_type(dtype) -> str`: BIGINT / DOUBLE PRECISION / TIMESTAMP / BOOLEAN / TEXT (moved here; still importable from `data_indexing`, as are `DATA_DB` and `INDEXED_FORMATS`). @@ -167,6 +168,7 @@ No file contents or values appear in the report. 2. Off-peak: `python manage.py migrate_data_table_names --report dry.json`. Review `counts`, `shared_groups`, `needs_manual_fix` (restore files first if you can), `unverified_tables` and `orphans`. 3. `python manage.py migrate_data_table_names --apply --report apply1.json`. Rerun until it prints "nothing to do". - `skipped_lock` and `failed` rows are retried. + - A row that comes back `failed` on two runs in a row needs a person: its file is stored and in a supported format but won't load or switch (it doesn't parse, has only a header, or has a header over 255 characters). Find it under `rows` with outcome `failed`, using the resource id and the `data_index_failed` / `data_index_load_failed` log lines. Its old table keeps serving unmarked, and a long one may hold another resource's data. - A shared table whose file is missing is marked on the run after its sharers move. 4. At least an hour later: `python manage.py migrate_data_table_names --report orphans.json`. Review the orphan list and remove from the file any table you want kept. 5. `python manage.py migrate_data_table_names --apply --drop-orphans --from-report orphans.json --report drop.json`. @@ -241,6 +243,7 @@ How the setup works: - **Uploads:** a first upload creates the row, the table, the column rows and the comment. A re-upload switches to a new table, drops the old one and keeps custom descriptions. - **Switch timing:** at the moment of the switch the new table is full and the old one still exists, and the lock is taken inside the switch transaction. The old table is dropped only after commit, with no transaction open and the new row saved. - **Failures keep the old data:** a value out of range mid-copy keeps the old row and table, leaves no new table, and logs error code 22003 without the value. A 300-character header fails the switch, keeps the old row, table and column rows, and drops the new table. +- **Stopped runs:** a run stopped (KeyboardInterrupt) while the old table's after-commit drop waits keeps the new table the resource already points at. The clean-up keeps a table whose row is saved, drops one whose row isn't, and leaves it when the check fails. - **Readers and races:** a reader still holding the old name gets "does not exist", never new rows. A slower build of an older file backs off when a newer file was saved, and the newer one stays. - **Transaction and default:** indexing inside an outer transaction is refused and builds nothing. A re-upload keeps the row's `created` and the dataset's default table in the data API. - **Deletes:** deleting a resource drops its table; deleting a dataset drops all its tables; a rolled-back delete keeps the table. Two legacy rows sharing a table: the table is kept until the last one is deleted. A drop of a table that's already gone doesn't fail. A drop blocked past the lock timeout is logged and the delete still succeeds. @@ -274,7 +277,7 @@ How the setup works: ### Layer 3 — Flow tests - **Upload flow order:** load → name → build → lock → file check → retire → save → rebuild columns → confirm. -- **Clean-up:** this build's table is dropped when the build, lock, retire or column rebuild fails, on a newer file, when the row is missing after commit, and on `KeyboardInterrupt`. It is never dropped after a confirmed switch. +- **Clean-up:** the unswitched-build clean-up runs when the build, lock, retire or column rebuild fails, on a newer file, when the row is missing after commit, and on `KeyboardInterrupt`. It never runs after a confirmed switch. - **Nothing to index:** nothing is built when inside a transaction, when there is no indexable file, or when nothing loaded. - **Receiver:** schedules one guarded drop per deleted row, via `on_commit(robust=True)`. - **Command:** the safety check runs first; a dry run changes nothing and writes the report; `--apply` runs row updates → renames → rebuilds (in plan order) → re-indexes; a failed safety check stops before any read; orphans are dropped only with `--apply` and a reviewed report. diff --git a/api/utils/data_tables.py b/api/utils/data_tables.py index 776967f0..3e2f8eb4 100644 --- a/api/utils/data_tables.py +++ b/api/utils/data_tables.py @@ -359,6 +359,29 @@ def drop_data_table(table_name: str) -> bool: return True +def drop_unswitched_table(row: ResourceDataTable) -> None: + """Drop the table of a build whose switch wasn't confirmed, unless its row was saved after all. + + The switch can commit and still not be confirmed: a run stopped (Ctrl+C) + while the old table's after-commit drop was waiting, or a commit that went + through but reported an error. So the row is checked first; if it exists, + the table is the resource's live data and stays. If the check itself fails, + the table is left for the migration command's orphan audit. Never raises + an Exception. + """ + try: + saved = data_table_row_exists(row.pk) + except Exception as error: + logger.error( + "data_index_switch_check_failed", table_name=row.table_name, **error_fields(error) + ) + return + if saved: + logger.warning("data_index_switch_saved_after_all", table_name=row.table_name) + return + drop_data_table(row.table_name) + + def drop_data_table_if_unshared(table_name: str, row_id: Any) -> None: """Drop a deleted row's table, unless another row still uses the same physical table. diff --git a/tests/data_index/test_reindex_switch.py b/tests/data_index/test_reindex_switch.py index 52b9f13d..db6ef8e0 100644 --- a/tests/data_index/test_reindex_switch.py +++ b/tests/data_index/test_reindex_switch.py @@ -28,6 +28,8 @@ from api.utils.data_tables import ( FileIdentity, LoadedFile, + build_data_table, + drop_unswitched_table, has_indexable_file, load_resource_frame, rebuild_resource_schema, @@ -182,6 +184,46 @@ def test_a_reader_still_holding_the_old_name_gets_does_not_exist_not_new_rows( cursor.execute(f'SELECT * FROM "{stale.table_name}"') +@uses_data_db +def test_a_run_stopped_while_the_old_table_is_dropped_keeps_the_new_table( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + + def stopped_by_the_operator(table_name: str, row_id: Any) -> None: + raise KeyboardInterrupt # Ctrl+C while the old table's drop waits for a reader + + with patch( + "api.signals.data_table_signals.drop_data_table_if_unshared", stopped_by_the_operator + ): + with pytest.raises(KeyboardInterrupt): + index_resource_data(resource) + + current = ResourceDataTable.objects.get(resource=resource) + assert current.pk != old.pk # the switch had committed + assert tables.exists(current.table_name) + assert len(tables.rows(current.table_name)) == 4 + + +@uses_data_db +def test_dropping_an_unswitched_build_checks_the_row_first(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + saved = index_resource_data(resource) + unsaved = data_tables.new_data_table_row(resource) + build_data_table(unsaved.table_name, pd.DataFrame({"a": [1]}), "{}") + + drop_unswitched_table(saved) + drop_unswitched_table(unsaved) + assert tables.exists(saved.table_name) + assert not tables.exists(unsaved.table_name) + + with patch("api.utils.data_tables.data_table_row_exists", side_effect=RuntimeError("db gone")): + drop_unswitched_table(saved) # can't tell: leave it for the orphan audit + assert tables.exists(saved.table_name) + + @uses_data_db def test_a_slower_build_of_an_older_file_backs_off_when_a_newer_file_was_saved( add_resource: Any, tables: Any @@ -319,7 +361,7 @@ def test_rebuild_columns_refuses_a_header_longer_than_the_column_list_allows( "save_data_table_row", "rebuild_resource_schema", "data_table_row_exists", - "drop_data_table", + "drop_unswitched_table", ] FILE = FileIdentity("resources/cases.csv", "abc123") @@ -385,7 +427,7 @@ def test_flow_drops_its_table_when_a_step_fails(flow: Any, helper: str) -> None: assert index_resource_data(MagicMock()) is None - flow.drop_data_table.assert_called_once_with("rdt_new") + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) def test_flow_drops_its_table_when_a_newer_file_was_saved(flow: Any) -> None: @@ -394,7 +436,7 @@ def test_flow_drops_its_table_when_a_newer_file_was_saved(flow: Any) -> None: assert index_resource_data(MagicMock()) is None flow.retire_data_table_row.assert_not_called() - flow.drop_data_table.assert_called_once_with("rdt_new") + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) def test_flow_drops_its_table_when_the_new_row_is_missing_after_the_commit(flow: Any) -> None: @@ -402,7 +444,7 @@ def test_flow_drops_its_table_when_the_new_row_is_missing_after_the_commit(flow: assert index_resource_data(MagicMock()) is None - flow.drop_data_table.assert_called_once_with("rdt_new") + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) def test_flow_drops_its_table_even_when_interrupted(flow: Any) -> None: @@ -411,12 +453,12 @@ def test_flow_drops_its_table_even_when_interrupted(flow: Any) -> None: with pytest.raises(KeyboardInterrupt): index_resource_data(MagicMock()) - flow.drop_data_table.assert_called_once_with("rdt_new") + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) def test_flow_keeps_the_table_after_a_confirmed_switch(flow: Any) -> None: index_resource_data(MagicMock()) - flow.drop_data_table.assert_not_called() + flow.drop_unswitched_table.assert_not_called() @pytest.mark.parametrize("stop_at", ["inside_transaction", "no_file", "nothing_loaded"]) @@ -431,4 +473,4 @@ def test_flow_builds_nothing_when_there_is_nothing_to_index(flow: Any, stop_at: assert index_resource_data(MagicMock()) is None flow.build_data_table.assert_not_called() - flow.drop_data_table.assert_not_called() + flow.drop_unswitched_table.assert_not_called() From 0e339eb53e1b0d31f97ac5dc9139616126e94130 Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 15:42:21 +0530 Subject: [PATCH 52/55] docs(data-index): mark the cut-column-name collision as a known TODO Two headers that differ only after Postgres's 63-byte cut become one column, so the file never gets a table. This bug predates per-upload tables and is noted, not fixed, by user decision. --- api/utils/data_indexing_architecture.md | 2 +- api/utils/data_tables.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/utils/data_indexing_architecture.md b/api/utils/data_indexing_architecture.md index 5b82e7d2..9eab48ed 100644 --- a/api/utils/data_indexing_architecture.md +++ b/api/utils/data_indexing_architecture.md @@ -336,7 +336,7 @@ n/a — all assertions are deterministic. ## Limitations & future work - **A failed re-upload still reports success to the uploader.** This is existing behaviour: the mutations ignore the indexing result. Now the old data stays served while the download serves the new file, until a successful re-upload. Surfacing the failure would change the upload API's contract and is left for a decision. -- **Two long headers that differ only after 63 bytes** become one column in Postgres, so the build fails. Devanagari headers hit this at about 21 characters. This existed before and is not fixed here. +- **TODO — two long headers that differ only after 63 bytes** become one column in Postgres, so the build fails and that file never gets a table (the upload still reports success; see the first item). Devanagari headers hit this at about 21 characters. This existed before and is not fixed yet. A fix would catch names that only clash once cut to 63 bytes during column clean-up, and either shorten them uniquely (keeping the stored column names in step) or refuse the file with a clear reason. A `TODO` comment marks the spot in `_clean_column_names` (`data_tables.py`). - **Access-model column selections are reset** whenever column rows are rebuilt (every re-upload, and migration rebuilds). Carrying them over by column name would be a small follow-up. - **Readers still use f-string table names.** That is safe for `rdt_` names; they were left as they are deliberately. - The plan's failure table said 5 s for the rename lock timeout. The design and the code use 2 s for renames and 5 s for drops. diff --git a/api/utils/data_tables.py b/api/utils/data_tables.py index 3e2f8eb4..9b810f7e 100644 --- a/api/utils/data_tables.py +++ b/api/utils/data_tables.py @@ -185,6 +185,9 @@ def on_alarm(signum: int, frame: Any) -> None: def _clean_column_names(columns: List[Any]) -> List[str]: + # TODO: Postgres keeps only the first 63 bytes of a column name, so two headers that + # differ only after that (about 21 Devanagari characters) become one column and the + # file never gets a table. Known, not fixed yet: data_indexing_architecture.md, Limitations. cleaned = [str(c).replace('"', "").replace("\n", " ").replace("\t", " ") for c in columns] seen: Counter = Counter() unique = [] From da795518c781fee93c702a1d66bed1626aac700c Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 15:54:00 +0530 Subject: [PATCH 53/55] docs(data-index): mark the silent failed re-upload as a known TODO A re-upload whose rows can't be loaded still reports success, so the download serves the new file while previews, charts and the data API serve the old one. Marked pending a product decision: enforce the database's limits at upload and flag problem columns in the column-description UI. --- api/schema/resource_schema.py | 6 ++++++ api/utils/data_indexing_architecture.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/api/schema/resource_schema.py b/api/schema/resource_schema.py index 4890fd0b..42f189b1 100644 --- a/api/schema/resource_schema.py +++ b/api/schema/resource_schema.py @@ -184,6 +184,12 @@ def _validate_file_details_and_update_format(resource: Resource) -> None: def _create_file_resource_schema(resource: Resource) -> None: """Create file resource schema.""" + # TODO: a failed indexing (the file saved, but its rows couldn't be loaded) still + # reports success, so the download serves the new file while previews, charts and + # the data API keep the old one. Pending a product decision: enforce the database's + # limits at upload (e.g. column names within 255 characters) and flag problem + # columns where people add column descriptions. See + # api/utils/data_indexing_architecture.md, Limitations. # Try to index CSV data if applicable data_table = index_resource_data(resource) diff --git a/api/utils/data_indexing_architecture.md b/api/utils/data_indexing_architecture.md index 9eab48ed..adbc6388 100644 --- a/api/utils/data_indexing_architecture.md +++ b/api/utils/data_indexing_architecture.md @@ -335,7 +335,7 @@ n/a — all assertions are deterministic. | Test data tables leak between tests | An autouse fixture drops every `public` table after each `data_db` test. | ## Limitations & future work -- **A failed re-upload still reports success to the uploader.** This is existing behaviour: the mutations ignore the indexing result. Now the old data stays served while the download serves the new file, until a successful re-upload. Surfacing the failure would change the upload API's contract and is left for a decision. +- **TODO — a failed re-upload still reports success to the uploader.** This is existing behaviour: the upload mutations ignore the indexing result. The file is saved (the download serves it), but if its rows can't be loaded, previews, charts and the data API keep serving the previous file, until a successful re-upload. Nobody is told the two disagree. Direction (pending a talk with the product team, with the DataSpace team): check the file at upload against the database's limits and enforce them on the platform (for example, column names within the 255 characters a column row allows), and flag problem columns where people see column names and add column descriptions. A `TODO` comment marks the spot in `_create_file_resource_schema` (`api/schema/resource_schema.py`). - **TODO — two long headers that differ only after 63 bytes** become one column in Postgres, so the build fails and that file never gets a table (the upload still reports success; see the first item). Devanagari headers hit this at about 21 characters. This existed before and is not fixed yet. A fix would catch names that only clash once cut to 63 bytes during column clean-up, and either shorten them uniquely (keeping the stored column names in step) or refuse the file with a clear reason. A `TODO` comment marks the spot in `_clean_column_names` (`data_tables.py`). - **Access-model column selections are reset** whenever column rows are rebuilt (every re-upload, and migration rebuilds). Carrying them over by column name would be a small follow-up. - **Readers still use f-string table names.** That is safe for `rdt_` names; they were left as they are deliberately. From 0dc798bf09bbe5398ff65033339ae1b0aa4db4eb Mon Sep 17 00:00:00 2001 From: psaesha Date: Sat, 14 Nov 2026 15:54:00 +0530 Subject: [PATCH 54/55] ci(sdk): run the SDK tests on ubuntu-24.04 ubuntu-latest is now Ubuntu 26.04, where setup-python has no Python 3.9 build. The 3.9 job failed at setup and fail-fast cancelled 3.10 and 3.11 (seen on #295). The SDK still declares 3.9 support, so pin the runner instead of dropping it. --- .github/workflows/sdk-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 672eec48..b504bc21 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -8,7 +8,8 @@ on: jobs: test: - runs-on: ubuntu-latest + # Pinned: ubuntu-latest (26.04) has no Python 3.9 build, which the SDK still supports. + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11"] From 4441862fde77bc104ac392b68ecc2e22e0b28659 Mon Sep 17 00:00:00 2001 From: Saqib Date: Sat, 14 Nov 2026 21:59:05 +0530 Subject: [PATCH 55/55] ci: run provider tests in the dev deploy gate again They were skipped while CivicDataSpace-test#103 was open; it closed on 2026-09-24 and the provider suite now passes under parallel load. --- .github/workflows/deploy-backend.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index efb1001b..d774740b 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -274,7 +274,6 @@ jobs: # force_smoke_failure test path. deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} - skip_provider: true # main: only the readonly allowlist ever runs against prod. # dev: stage 1 of 2 -- functional-tests runs only if this passes. suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }} @@ -309,7 +308,6 @@ jobs: with: suite: functional api_base_url: ${{ vars.DEV_API_BASE_URL }} - skip_provider: true secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }}