diff --git a/.env.example b/.env.example index 7887793e..3de8e087 100644 --- a/.env.example +++ b/.env.example @@ -21,15 +21,18 @@ GITHUB_TOKEN= # URLs written into exported metadata (DCAT / Croissant); set per environment PUBLIC_SITE_URL=https://civicdataspace.in -PUBLIC_API_URL=https://api.civicdataspace.in +PUBLIC_API_URL=https://api.datakeep.civicdays.in -# --- s3 only --- -# STORAGE_BUCKET=dataspace-dev -# STORAGE_REGION=ap-south-1 -# STORAGE_ENV=dev -# STORAGE_PUBLIC_BASE_URL= -# STORAGE_ACCESS_KEY_ID= -# STORAGE_SECRET_ACCESS_KEY= +# Media storage (S3 only; required). STORAGE_ENV prefixes every key: use +# "local" for local development against the dev bucket. Leave the access keys +# empty on ECS/EC2 so the task role is used. STORAGE_PUBLIC_BASE_URL is the +# base for public asset URLs (CDN or bucket URL); empty uses the S3 bucket URL. +STORAGE_BUCKET=dataspace-dev +STORAGE_REGION=ap-south-1 +STORAGE_ENV=local +STORAGE_PUBLIC_BASE_URL= +STORAGE_ACCESS_KEY_ID= +STORAGE_SECRET_ACCESS_KEY= # Per-IP request throttling (api/middleware/rate_limit.py). False on dev only. RATE_LIMIT_ENABLED=True diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 85158cec..d774740b 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -64,6 +64,11 @@ on: type: boolean required: false default: false + rollback: + description: "Roll dev back to the image that was live before the last deploy (no build, no tests). Ignored on main." + type: boolean + required: false + default: false # Queue overlapping deploys to the SAME environment rather than cancelling (a # cancelled run mid-deploy could leave .deploy/ state and the running @@ -82,6 +87,7 @@ env: jobs: build: name: Build and push image + if: ${{ !inputs.rollback }} runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -179,6 +185,9 @@ jobs: -e SECRET_KEY=ci-sanity-check-not-a-real-key \ -e URL_WHITELIST=http://localhost \ -e DB_ENGINE=django.db.backends.sqlite3 \ + -e STORAGE_BUCKET=ci-sanity-check \ + -e STORAGE_REGION=ap-south-1 \ + -e STORAGE_ENV=ci \ --entrypoint python \ "${{ steps.ref.outputs.image_ref }}" \ manage.py check; then @@ -250,6 +259,9 @@ jobs: smoke-tests: name: Smoke Tests needs: deploy + # Repo variable SKIP_DEV_TESTS=true skips the dev test gates (e.g. while the + # test owner is away). main ignores it: prod deploys are always gated. + if: github.ref_name == 'main' || vars.SKIP_DEV_TESTS != 'true' # No `environment:` here -- GitHub rejects the entire workflow file at # parse time if a `uses:` job declares one. Consequence: this job also # cannot see environment-scoped vars, which is why api_base_url comes @@ -262,7 +274,6 @@ jobs: # force_smoke_failure test path. deployed_sha: ${{ (inputs.force_smoke_failure == true && 'forced-failure-sentinel') || github.sha }} min_passed: ${{ inputs.force_smoke_failure && 999 || 1 }} - skip_provider: true # main: only the readonly allowlist ever runs against prod. # dev: stage 1 of 2 -- functional-tests runs only if this passes. suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }} @@ -276,6 +287,12 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a # confidential client, so without this the token request returns 401 # and the job fails its preflight. @@ -291,24 +308,41 @@ jobs: with: suite: functional api_base_url: ${{ vars.DEV_API_BASE_URL }} - skip_provider: true secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + tests-skipped: + name: Tests skipped (SKIP_DEV_TESTS) + needs: deploy + if: github.ref_name != 'main' && vars.SKIP_DEV_TESTS == 'true' + runs-on: ubuntu-latest + steps: + - run: | + echo "::notice::SKIP_DEV_TESTS=true: smoke and functional tests were skipped for this dev deploy." + echo "**SKIP_DEV_TESTS=true**: smoke and functional tests were skipped. Delete the repo variable to turn them back on." >> "$GITHUB_STEP_SUMMARY" + rollback-on-smoke-failure: name: Rollback (smoke tests failed) # `deploy` must be in needs: for needs.deploy.result to resolve here. needs: [deploy, smoke-tests, functional-tests] # Explicit results rather than failure(), so that a skipped functional stage # on main never reads as a failure. !cancelled() keeps cancellation a no-op. + # Also the manual rollback button: Run workflow on dev with `rollback` ticked. if: >- - ${{ !cancelled() && needs.deploy.result == 'success' && - (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }} + ${{ !cancelled() && ((inputs.rollback && github.ref_name != 'main') || + (needs.deploy.result == 'success' && + (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure'))) }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} # 50m: must comfortably exceed the Restore previous image step's own @@ -347,6 +381,7 @@ jobs: script: cd "$HOME/${{ env.DEPLOY_PATH }}" && sudo bash scripts/ci-rollback.sh "${{ vars.HEALTH_CHECK_URL || 'http://127.0.0.1:8000/health/' }}" "${{ github.actor }}" - name: Mark this run as failed + if: ${{ !inputs.rollback }} # The mitigation succeeded, but the run must still read RED -- a bad # deploy that silently self-heals is a bad deploy nobody investigates. run: | @@ -357,10 +392,10 @@ jobs: name: Finalize Deploy needs: [deploy, smoke-tests, functional-tests] # Explicit results, not success()/!failure(): on main the functional stage - # is skipped by design. + # is skipped by design, and on dev SKIP_DEV_TESTS skips smoke too. if: >- ${{ !cancelled() && needs.deploy.result == 'success' && - needs.smoke-tests.result == 'success' && + (needs.smoke-tests.result == 'success' || needs.smoke-tests.result == 'skipped') && (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }} runs-on: ubuntu-latest environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} diff --git a/.github/workflows/deploy-to-ecs.yml b/.github/workflows/deploy-to-ecs.yml index f3573b94..67603e97 100644 --- a/.github/workflows/deploy-to-ecs.yml +++ b/.github/workflows/deploy-to-ecs.yml @@ -158,6 +158,12 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} # api-smoke authenticates against Keycloak via ROPC. `dataspace` is a # confidential client, so without this the token request returns 401 # and the job fails its preflight. diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 6bca8c20..2f49c740 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -16,6 +16,20 @@ jobs: django-tests: runs-on: ubuntu-latest + # The data database: tests marked data_db build real tables in it. + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: - uses: actions/checkout@v4 @@ -40,6 +54,10 @@ jobs: # tests/object_types/charts and the dataset metadata tests were left # behind by the chart consolidation and fail on main: see #189. - name: Run tests + env: + TEST_DATA_DB_URL: postgres://postgres:postgres@localhost:5432/postgres + # Fail the run instead of skipping the data_db tests if the database is missing. + REQUIRE_DATA_DB_TESTS: "1" run: | pytest tests/ \ --ignore=tests/object_types/charts \ diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 5a9f6ce3..98829b06 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -48,4 +48,10 @@ jobs: TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + TEST_EMAIL_3: ${{ secrets.TEST_EMAIL_3 }} + TEST_PASSWORD_3: ${{ secrets.TEST_PASSWORD_3 }} + TEST_EMAIL_4: ${{ secrets.TEST_EMAIL_4 }} + TEST_PASSWORD_4: ${{ secrets.TEST_PASSWORD_4 }} + TEST_EMAIL_5: ${{ secrets.TEST_EMAIL_5 }} + TEST_PASSWORD_5: ${{ secrets.TEST_PASSWORD_5 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 672eec48..b504bc21 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -8,7 +8,8 @@ on: jobs: test: - runs-on: ubuntu-latest + # Pinned: ubuntu-latest (26.04) has no Python 3.9 build, which the SDK still supports. + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11"] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..3f7f4bea --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,282 @@ +# AGENTS.md + +Instructions for AI coding agents (and humans) working on the DataSpace backend. +Read this file first; the rules here override general habits. + +You are a senior engineer on this codebase: +- You write clean, reusable, maintainable code that reads like the code around it. +- You know its security risks (authorization, multi-tenant organization data, file uploads, + search exposure) and you turn them into clear, actionable findings. +- You ship small, well-tested changes a teammate can merge with confidence, and explain *why* + when the reason isn't obvious. + +## Quick start + +```bash +cp .env.example .env # settings fail without it (e.g. URL_WHITELIST, STORAGE_*) +mkdir -p logs # the "file" log handler writes logs/dataex.log +pip install -r requirements.txt +pip install pytest pytest-django # test-only, not in requirements.txt +``` + +- **Run (Docker, hot reload):** `docker compose -f docker-compose.yml -f docker-compose.hotreload.yml up --build` +- **Run (venv):** `python manage.py runserver` + +## Commands + +| Task | Command | +|---|---| +| Tests (as CI runs them) | `pytest tests/ --ignore=tests/object_types/charts --deselect tests/schema/test_dataset_schema.py::TestAddUpdateDatasetMetadata` | +| Single test | `pytest tests/test_slug_collision.py -k usecase` | +| SDK tests (no Django) | `pytest tests/test_auth.py tests/test_base.py tests/test_client.py tests/test_datasets.py tests/test_aimodels.py tests/test_usecases.py tests/test_exceptions.py -p no:django -o addopts= --override-ini=django_find_project=false --noconftest` | +| All hooks | `pre-commit run --all-files` (whitespace, black, isort, mypy, `debug-statements`) | +| Format | `black .` and `isort --profile black .` (line length 100, from `pyproject.toml`) | +| Type check | `mypy --config-file=mypy.ini .` | +| Migrations | `python manage.py makemigrations api`, then `python manage.py migrate` | +| Search mappings | `python manage.py sync_search_mappings` after any change in `search/documents/` | +| API testing | Postman CLI; collections live in `postman/collections/` | + +Notes: +- Tests use SQLite in memory with `--nomigrations` (`pytest.ini`, settings in + `tests/test_settings.py`). All media storages are `InMemoryStorage`, so tests never reach S3. +- The CI exclusions above are known-broken tests (#189). Don't add to them. +- pre-commit hooks pin `python3.11`. If it isn't installed, pre-commit can't build its + environments: run black, isort and mypy from the venv on the files you changed, and say so. +- mypy results depend on its cache. To tell new errors from old ones, compare runs with a fresh + `--cache-dir`, or check only errors in the files you touched. + +## Definition of done + +A change is finished only when: +1. `pre-commit run --all-files` is clean (or the fallback above is clean for the touched files). +2. Tests pass with the CI flags, and a bug fix has a regression test. +3. Every new or changed GraphQL query/mutation has the right permission guard (see Security). +4. If a model changed, its migration, search document, signals, cache invalidation, GraphQL type + and SDK resource all still match (see "Changing a model"). + +## Project knowledge + +**What it is:** DataSpace is CivicDataLab's platform for publishing and discovering datasets, +use cases, AI models, collaboratives and publications. This repo is the backend: a GraphQL API +(main surface) plus REST endpoints for search, downloads, auth and activity. It also ships a +Python client, `dataspace_sdk`, published to PyPI. + +**Tech stack:** +- Python 3.11, Django 5.0, PostgreSQL +- GraphQL: `strawberry-graphql` 0.235 + `strawberry-graphql-django` 0.42 (endpoint `/api/graphql`) +- REST: Django REST Framework 3.14, `drf-yasg` for API docs +- Auth: Keycloak (`python-keycloak` 5.5). Every request needs a real Keycloak JWT; there is no dev-mode fallback. +- Search: Elasticsearch 8 via `django-elasticsearch-dsl` +- Cache: Redis (`django-redis`, `django-cacheops`) +- Activity feed: `django-activity-stream` +- Observability: OpenTelemetry, `structlog` +- Data: pandas, pyarrow +- Media: S3 only, via `django-storages` (`public`/`private` storages in `api/utils/storage.py`; keys prefixed by `STORAGE_ENV`) +- Runtime: uvicorn/gunicorn in Docker, deployed to AWS (ECS / EC2 via `scripts/ci-*.sh`) + +**Layout:** +- `DataSpace/`: Django project (`settings.py`, `urls.py`, `asgi.py`, `cache_settings.py`) +- `api/`: main app + - `models/`: one model per file (`Dataset.py`, `UseCase.py`, …), exported from `models/__init__.py` + - `schema/`: Strawberry `Query`/`Mutation` classes per domain (`*_schema.py`), merged with + `merge_types` in `schema/schema.py`. `base_mutation.py` holds `BaseMutation`/`MutationResponse`; + `extensions.py` holds `TrackActivity`/`TrackModelActivity`. + - `types/`: Strawberry output types (`type_*.py`) + - `views/`: REST views (search, downloads, charts, health, auth, legacy media redirect) + - `services/`: business logic kept out of resolvers (publication, platform import, metadata mapping/export, model clients) + - `activities/`: activity-stream handlers and display per domain, wired up in `registry.py` + - `signals/`: post-save hooks per domain (search indexing, cache invalidation) + - `middleware/`, `utils/middleware.py`: request validation, rate limiting, structured logging, organization/dataspace context from headers + - `utils/`: shared helpers (`graphql_utils.py`, `keycloak_utils.py`, `storage.py`, `file_utils.py`, `file_paths.py`, `search_query.py`, `search_cache.py`, `enums.py`, …) + - `management/commands/`: `sync_search_mappings`, `update_dataindex`, `migrate_media_to_s3`, `seed_resource_types`, … + - `migrations/`: numbered sequentially (`00NN_*`) +- `authorization/`: Keycloak integration, roles, `OrganizationMembership`, consent/activity middleware + - `permissions.py`: both the DRF permission classes and the Strawberry `*GraphQL` ones + - `schema/`: user/membership GraphQL schema; `schema/permissions.py` holds `HasOrganizationAdminRole` + - `graphql_permissions.py`: legacy duplicate of the GraphQL classes; don't add new uses +- `search/documents/`: Elasticsearch document definitions and analysers +- `dataspace_sdk/`: standalone Python SDK (`client.py`, `resources/.py`; own version in `pyproject.toml`; depends only on `requests`) +- `tests/`: pytest suite (`test_*.py`, `tests/schema/`, `tests/journeys/`). `test_auth.py`, `test_base.py`, `test_client.py`, `test_datasets.py`, `test_aimodels.py`, `test_usecases.py` and `test_exceptions.py` test the SDK. +- `docs/`: feature docs (Keycloak, unified search, charts, activity stream, SDK) +- `postman/`: Postman collections and fixtures +- `aws/`, `scripts/`, `.github/workflows/`: infra, deploy scripts, CI +- Root-level `migrate_*.py`, `populate_sdgs.py`, `test_geography_hierarchy.py`: one-off scripts. + Leave them alone; new scripts go in `api/management/commands/`. + +## Code standards + +**Naming:** +- Functions, methods, variables: snake_case. Classes: PascalCase. Constants: UPPER_SNAKE_CASE. +- Strawberry output types: `Type` in `api/types/type_.py`. +- Model files: PascalCase, one model per file (`api/models/UseCase.py`). +- GraphQL fields are snake_case in Python; Strawberry exposes them as camelCase + (`create_organization` → `createOrganization`). + +**Style:** +- Type-annotate every function. Don't add `# type: ignore` or `mypy.ini` exemptions without a + reason in a comment. The existing `# type: ignore[list-item]` on `permission_classes` lists that + mix in `HasOrganizationRoleGraphQL` subclasses is the accepted pattern. + mypy skips `tests/` and is relaxed for `api.models`, `api.activities` and the resource, use case + and dataset schemas; still annotate new code there. +- Keep resolvers thin. Put reusable logic in `api/services/` or `api/utils/`, and reuse existing + helpers before writing new ones. +- Wrap multi-step writes in `transaction.atomic()`. +- Return clean client errors (400/404, or `MutationResponse.error_response`). Never let + `IntegrityError`, `ValueError` or `DoesNotExist` surface as a 500, and never return raw + exception text to clients. +- Log with the module's existing `structlog` logger. Never log tokens, secrets or PII. +- Comments explain *why* (constraints, past incidents), not *what*. Match the density of the + surrounding file. +- Make the smallest change that solves the problem. Don't reformat, rename or refactor + unrelated code in the same change. + +## Security + +- Every GraphQL query and mutation that touches non-public data needs `permission_classes`. + Don't rely on the ORM to reject anonymous users. + - Import from `authorization.permissions`: `IsAuthenticated`, `IsOrganizationMemberGraphQL`, + `HasOrganizationRoleGraphQL`, or a domain class built on them (`CreateDatasetPermission`, + `ChangePublicationPermission`, the `Change/DeleteOrganizationPermission` subclasses in + `organization_schema.py`, …). + - The plain `IsOrganizationMember` and `HasOrganizationRole` in `authorization/permissions.py` + are DRF classes for REST views and don't work in Strawberry `permission_classes`. Schemas + alias the GraphQL versions instead (`HasOrganizationRoleGraphQL as HasOrganizationRole`). + - `HasOrganizationRoleGraphQL` checks the organization from the request header (or an + `organization_id` argument), **not** the object an `input.id` points at. When a mutation loads + an object by id, also check the caller's access to that object's organization/dataset inside + the resolver (e.g. `AuthorizationService.check_dataset_permission`). +- Scope querysets to what the user may see (organization membership, dataspace, published + status). Superuser bypass goes through `is_superuser(info)` from `api/utils/graphql_utils.py`. +- Search must not expose non-public objects. Documents gate this in `should_index_object`; keep + that check in sync when you add a status or visibility field. + +## Files and media + +- Media lives only in S3. Every `FileField`/`ImageField` sets `storage=public_storage` or + `storage=private_storage` from `api/utils/storage.py`. Use **private** unless the file is meant + to be world-readable (logos, cover images, profile pictures, published publication files). +- Never use `FieldFile.path`: S3 has no local path (and tests use `InMemoryStorage`). Use + `file.open("rb")`/`file.chunks()`, `file.storage.exists(file.name)`, or + `api.utils.file_utils.local_copy(file)` for libraries that need a real path (pandas, libmagic, + YAML/XML parsers). +- Serve downloads with `FileResponse(file.open("rb"))`, not `HttpResponse(file.read())`. +- `upload_to` helpers in `api/utils/file_paths.py` return names relative to the storage; don't + prefix them with `files/` or `public/`. +- Validate uploads with the domain's existing validator before saving (`file_validation` in + `api/utils/file_utils.py` for resources, `validate_publication_file` in + `api/utils/publication_uploads.py` for publication blocks). Don't trust the extension alone. + +## Task checklists + +**Adding or changing a GraphQL mutation** +1. Put it in the domain's `api/schema/_schema.py`; logic in `api/services/` if non-trivial. +2. Add `permission_classes` and an object-level check in the resolver when it loads by id. +3. Add `TrackModelActivity`/`TrackActivity` if the action belongs in the activity feed. +4. Test the allowed caller **and** a denied caller (anonymous, non-member, wrong organization). + +**Changing a model** (ask first: see Boundaries) +1. Edit `api/models/.py`, then `python manage.py makemigrations api` and review the file. +2. Update the `Type` in `api/types/` and any input types in the schema. +3. Update `search/documents/` if the field is indexed, then `sync_search_mappings`. +4. Check `api/signals/` and cache invalidation still cover the change. +5. Update the SDK resource in `dataspace_sdk/resources/` if the field is exposed through the API. + +**Fixing a bug** +1. Reproduce it with a failing test first. +2. Write the regression test in the style of `tests/test_slug_collision.py`: a module docstring + with the root cause and the fix, then small focused tests. +3. Fix the cause, not the symptom. + +## Code style example + +```python +from api.schema.extensions import TrackModelActivity +from api.utils.graphql_utils import is_superuser +from authorization.models import OrganizationMembership +from authorization.permissions import HasOrganizationRoleGraphQL as HasOrganizationRole +from authorization.permissions import IsAuthenticated + + +class ChangeOrganizationPermission(HasOrganizationRole): + def __init__(self) -> None: + super().__init__(operation="change") + + +# ✅ Good - permission guard, typed, scoped, atomic, clean error +@strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[IsAuthenticated, ChangeOrganizationPermission], # type: ignore[list-item] + extensions=[ + TrackModelActivity( + verb="updated", + get_data=lambda result, **kwargs: { + "organization_name": result.name, + "organization_id": str(result.id), + }, + ) + ], +) +def update_organization(self, info: Info, input: OrganizationInputPartial) -> TypeOrganization: + """Update an organization the caller administers.""" + try: + organization = Organization.objects.get(id=input.id) + except Organization.DoesNotExist: + raise ValueError(f"Organization with ID {input.id} does not exist.") + + # The permission class checks the organization in the request header, not input.id + is_admin = OrganizationMembership.objects.filter( + user=info.context.user, organization=organization, role__name="admin" + ).exists() + if not is_superuser(info) and not is_admin: + raise PermissionError("You don't have permission to update this organization.") + + with transaction.atomic(): + for field, value in vars(input).items(): + if field != "id" and value is not strawberry.UNSET: + setattr(organization, field, value) + organization.save() + return TypeOrganization.from_django(organization) + + +# ❌ Bad - no auth guard, untyped, unscoped lookup, raw 500 on a missing id +@strawberry_django.mutation +def update_org(self, info, input): + org = Organization.objects.get(id=input.id) + for k, v in vars(input).items(): + setattr(org, k, v) + org.save() + return org +``` + +## Git and PRs + +- Don't commit until asked. +- Branch from `dev` with `fix/…`, `feat/…` or `chore/…`. PRs go into `dev`. Releases go from + `dev` to `main`, gated by `pr-gate.yml`. +- Conventional Commits with a scope: `fix(api): …`, `feat(usecase): …`; mark breaking changes + with `!` and a `BREAKING CHANGE:` footer. The body says *why*, not a list of files. +- Split large work into commits that each build and pass tests on their own. +- Never add a Claude (or any AI) `Co-Authored-By` trailer to commits. +- Never `git stash`, `reset` or `checkout --` the user's uncommitted work. To compare against a + baseline, use `git worktree add HEAD` (copy `.env` and create `logs/` there). +- When a merge brings in migrations with clashing numbers, renumber yours after the latest one on + `dev` and fix the `dependencies`. + +## Boundaries + +- ✅ **Always:** work in `api/`, `authorization/`, `search/`, `dataspace_sdk/` and `tests/`. Run + the tests and hooks before saying a change is done. +- ⚠️ **Ask first:** + - model or schema changes, and new migrations + - changes to permission classes, Keycloak/auth middleware or rate limits + - changes to Elasticsearch mappings (these need a reindex) + - adding, removing or upgrading dependencies + - bumping the SDK version or publishing it + - touching `.github/workflows/`, `aws/`, `scripts/ci-*.sh`, `Dockerfile` or `docker-compose.yml` + - anything that runs against a deployed environment or a real S3 bucket +- 🚫 **Never:** + - commit `.env`, secrets, Keycloak client secrets or tokens. Use `.env.example` for new variables. + - edit or delete already-merged migrations. Add a new migration instead. + - disable an auth check, skip or delete a failing test to get green. Fix the cause, or ask. + - edit `.venv/`, `files/` (user uploads) or `logs/` + - force-push to `dev` or `main` diff --git a/DataSpace/settings.py b/DataSpace/settings.py index f3e4c0ab..75cb67d9 100644 --- a/DataSpace/settings.py +++ b/DataSpace/settings.py @@ -255,8 +255,39 @@ # https://docs.djangoproject.com/en/4.0/howto/static-files/ # This STATIC_URL setting is overridden below +# Uploaded media lives only in S3 (see STORAGES below). MEDIA_URL is kept as the +# prefix of the legacy /public/ redirect route for links issued before S3. MEDIA_URL = "public/" -MEDIA_ROOT = os.path.join(BASE_DIR, "files", "public") + +# S3 media storage. STORAGE_ENV prefixes every key (local, dev, prod) so one +# bucket can hold several environments. Empty keys fall back to the boto3 +# credential chain, which is what ECS/EC2 task roles use. +STORAGE_BUCKET = env("STORAGE_BUCKET") +STORAGE_REGION = env("STORAGE_REGION") +STORAGE_ENV = env("STORAGE_ENV") +STORAGE_PUBLIC_BASE_URL = env("STORAGE_PUBLIC_BASE_URL", default="") + +_S3_OPTIONS = { + "bucket_name": STORAGE_BUCKET, + "region_name": STORAGE_REGION, + "access_key": env("STORAGE_ACCESS_KEY_ID", default="") or None, + "secret_key": env("STORAGE_SECRET_ACCESS_KEY", default="") or None, + "file_overwrite": False, + # The bucket enforces object ownership; access is granted by bucket policy, not ACLs. + "default_acl": None, +} +_S3_PUBLIC_OPTIONS = { + **_S3_OPTIONS, + "location": f"{STORAGE_ENV}/public", + "querystring_auth": False, + "custom_domain": STORAGE_PUBLIC_BASE_URL.split("://", 1)[-1].rstrip("/") or None, +} +_S3_PRIVATE_OPTIONS = { + **_S3_OPTIONS, + "location": f"{STORAGE_ENV}/private", + "querystring_auth": True, + "querystring_expire": 300, +} # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field @@ -288,6 +319,7 @@ "search.documents.aimodel_document": "aimodel", "search.documents.publication_document": "publication", "search.documents.collaborative_document": "collaborative", + "search.documents.event_document": "event", "search.documents.publisher_document.OrganizationPublisherDocument": "organization_publisher", "search.documents.publisher_document.UserPublisherDocument": "user_publisher", } @@ -306,12 +338,7 @@ # Absolute URLs written into exported metadata documents (dataset landing page, # download links). Override on any environment that is not production. PUBLIC_SITE_URL = os.getenv("PUBLIC_SITE_URL", "https://civicdataspace.in") -PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.civicdataspace.in") - -# DVC settings -DVC_REPO_PATH = os.path.join(BASE_DIR, "dvc") -DVC_REMOTE_NAME = os.getenv("DVC_REMOTE_NAME", None) -DVC_REMOTE_URL = os.getenv("DVC_REMOTE_URL", None) +PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.datakeep.civicdays.in") # Django REST Framework settings REST_FRAMEWORK = { @@ -377,6 +404,9 @@ "level": "DEBUG" if DEBUG else "INFO", "propagate": True, }, + # At DEBUG the AWS SDK logs every signed request, Authorization header + # included, and floods the log on each S3 call. + **{name: {"level": "WARNING"} for name in ("boto3", "botocore", "s3transfer", "urllib3")}, }, } @@ -490,8 +520,16 @@ "whitenoise.middleware.WhiteNoiseMiddleware", ) -# Use the simplest WhiteNoise storage configuration to avoid MIME type issues -STATICFILES_STORAGE = "whitenoise.storage.StaticFilesStorage" +# "public"/"private" are what model fields name through api.utils.storage. The +# default is private so anything that falls back to default_storage is never +# world-readable by accident. +STORAGES = { + "default": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS}, + "public": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PUBLIC_OPTIONS}, + "private": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS}, + # Use the simplest WhiteNoise storage configuration to avoid MIME type issues + "staticfiles": {"BACKEND": "whitenoise.storage.StaticFilesStorage"}, +} # Disable compression to avoid MIME type issues WHITENOISE_ENABLE_COMPRESSION = False diff --git a/DataSpace/urls.py b/DataSpace/urls.py index 281b717e..85f149c5 100644 --- a/DataSpace/urls.py +++ b/DataSpace/urls.py @@ -23,7 +23,7 @@ from drf_yasg import openapi from drf_yasg.views import get_schema_view -from api.views import health +from api.views import health, legacy_media # Type alias for URL patterns URLPatternsList = List[Union[URLPattern, URLResolver]] @@ -49,16 +49,22 @@ # Health check endpoint path("health/", health.health_check, name="health_check"), # API documentation - path( - "swagger/", schema_view.without_ui(cache_timeout=0), name="schema-json" - ), + path("swagger/", schema_view.without_ui(cache_timeout=0), name="schema-json"), path( "swagger/", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui", ), path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"), -] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + # Links issued before media moved to S3; see api/views/legacy_media.py. + path( + # Django prefixes a relative MEDIA_URL with the script name ("/public/"), + # which a URL pattern never matches. + f"{settings.MEDIA_URL.lstrip('/')}", + legacy_media.legacy_media_redirect, + name="legacy_media", + ), +] # In debug mode, add static URL patterns and debug toolbar if settings.DEBUG: diff --git a/SECURITY.md b/SECURITY.md index e3caa67a..f3afb381 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -134,21 +134,20 @@ When contributing to this project, please follow these security guidelines: - **GraphQL**: Implements query complexity limits and depth restrictions - **REST API**: Rate limiting and throttling enabled - **File Uploads**: Validated for type, size, and content -- **Data Versioning**: DVC integration for data versioning and tracking ### Data Protection - **Encryption**: All data encrypted in transit (TLS 1.2+) - **Database**: PostgreSQL with proper access controls - **Search**: Elasticsearch with authentication enabled - **Caching**: Redis with secure configuration -- **File Storage**: Secure file handling and validation +- **File Storage**: Media in S3; public assets via unsigned URLs, private files via presigned URLs or gated views ### Third-Party Services - **Keycloak**: Ensure proper realm and client configuration - **Elasticsearch**: Enable security features and authentication - **Redis**: Use password protection and disable dangerous commands - **OpenTelemetry**: Configure to exclude sensitive data -- **DVC**: Secure data versioning and storage +- **S3**: Block public access except the `*/public/*` prefix; least-privilege task role ## Security Checklist for Pull Requests diff --git a/api/management/commands/backfill_file_hashes.py b/api/management/commands/backfill_file_hashes.py index 139af5dd..a97e5172 100644 --- a/api/management/commands/backfill_file_hashes.py +++ b/api/management/commands/backfill_file_hashes.py @@ -3,7 +3,7 @@ python manage.py backfill_file_hashes # fill every missing hash python manage.py backfill_file_hashes --dry-run # only count -Writes through a queryset update, so the DVC versioning signal does not run. +Writes through a queryset update, so no model save signals run. """ from django.core.management.base import BaseCommand diff --git a/api/management/commands/migrate_data_table_names.py b/api/management/commands/migrate_data_table_names.py new file mode 100644 index 00000000..d48e6a0d --- /dev/null +++ b/api/management/commands/migrate_data_table_names.py @@ -0,0 +1,166 @@ +"""Move existing data tables to per-upload names, and list the tables nothing uses. + + python manage.py migrate_data_table_names # dry run: write a report, change nothing + python manage.py migrate_data_table_names --apply # rename, rebuild, fix rows + python manage.py migrate_data_table_names --apply --drop-orphans --from-report + + --resource-id act on one resource only + --report where to write the JSON report (default: a timestamped file here) + +Short old names are renamed in place; long ones (which may hold another +resource's data) are rebuilt from each resource's own file. A long one whose +file is missing or unusable is renamed and marked "unverified" (listed in +every report until the resource is re-uploaded), or left alone if another row +shares it. A dry run checks the files too, so it shows what --apply will do. +Safe to rerun: each run picks up whatever is left ("nothing to do" once only +rows needing a person remain), and doubles as an orphan audit. + +Docs (runbook, failure model): ../../utils/data_indexing_architecture.md +""" + +from typing import Any, Dict, List + +from django.core.management.base import BaseCommand, CommandError, CommandParser +from django.utils import timezone + +from api.utils.data_table_migration import ( + access_models_touched, + build_report, + drop_reviewed_orphans, + find_orphans, + find_unverified, + nothing_left_to_do, + plan_table_migration, + point_row_at_its_table, + read_data_table_rows, + read_reviewed_orphans, + rebuild_decision, + rebuild_from_file, + record_outcome, + reindex_decision, + reindex_missing_table, + rename_legacy_table, + same_file, + summary_lines, + write_report, +) +from api.utils.data_tables import ( + DataDbNotSeparate, + assert_data_db_is_separate, + list_data_db_tables, +) + + +class Command(BaseCommand): + help = "Move data tables to per-upload rdt_ names; list (and drop reviewed) orphans." + + def add_arguments(self, parser: CommandParser) -> None: + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", help="Report only (the default).") + mode.add_argument("--apply", action="store_true", help="Rename, rebuild and fix rows.") + parser.add_argument( + "--drop-orphans", + action="store_true", + help="With --apply: drop orphans listed in --from-report that are still orphans.", + ) + parser.add_argument("--from-report", help="A dry-run report someone reviewed.") + parser.add_argument("--resource-id", help="Act on this resource only.") + parser.add_argument("--report", help="Where to write this run's JSON report.") + + def handle(self, *args: Any, **options: Any) -> None: + apply, drop_orphans, from_report = ( + options["apply"], + options["drop_orphans"], + options["from_report"], + ) + mode = "apply" if apply else "dry-run" + now = timezone.now() + report_path = options["report"] or f"data_table_migration_{now:%Y%m%dT%H%M%SZ}.json" + + # Orphans are dropped only by an applying run, and only from a reviewed report. + if drop_orphans and not (apply and from_report): + raise CommandError("--drop-orphans needs --apply and --from-report ") + if from_report and same_file(report_path, from_report): + raise CommandError("--report would overwrite the reviewed --from-report file") + + # Never touch anything if the data database might be the main database. + try: + assert_data_db_is_separate() + except DataDbNotSeparate as error: + raise CommandError(f"Refusing to run: {error}") from error + + # Read the reviewed orphan list now, so a bad file stops the run before any change. + reviewed = read_reviewed_orphans(from_report) if drop_orphans else set() + + # Write an in-progress report first: the path works, and a run that dies leaves a trace. + write_report(report_path, build_report(mode, now, status="in_progress")) + + # Read every row and every data table, and work out what each row needs. + plan = plan_table_migration( + read_data_table_rows(), list_data_db_tables(), now, resource_id=options["resource_id"] + ) + entries: List[Dict[str, Any]] = [ + record_outcome(row, "none", "already_migrated") for row in plan.already_migrated + ] + + if apply: + # Finish renames a crashed run left half done: just point the row at its table. + for row in plan.row_updates: + entries.append(record_outcome(row, "row_update", point_row_at_its_table(row))) + + # Rename short old tables in place; they can't have collided. + for row in plan.renames: + entries.append(record_outcome(row, "rename", rename_legacy_table(row))) + + # Rebuild long old tables from each resource's own file, oldest first. + for row, shared in plan.rebuilds: + outcome = rebuild_from_file(row, shared) + entries.append(record_outcome(row, "rebuild", outcome, shared=shared)) + + # Re-index rows whose table is gone, from their file. + for row in plan.missing_tables: + entries.append(record_outcome(row, "reindex", reindex_missing_table(row))) + else: + # A dry run changes nothing, but checks each stored file so the report shows + # what --apply would really do (rebuild, unverified rename, or leave for a person). + for row in plan.row_updates: + entries.append(record_outcome(row, "row_update", "planned_row_update")) + for row in plan.renames: + entries.append(record_outcome(row, "rename", "planned_rename")) + for row, shared in plan.rebuilds: + decision = rebuild_decision(row, shared) + entries.append(record_outcome(row, "rebuild", f"planned_{decision}", shared=shared)) + for row in plan.missing_tables: + entries.append(record_outcome(row, "reindex", f"planned_{reindex_decision(row)}")) + + # Rebuilt resources get new column rows, which resets access models' column choices. + touched = access_models_touched( + [row.resource_id for row, _ in plan.rebuilds] + + [row.resource_id for row in plan.missing_tables] + ) + + dropped: List[str] = [] + if drop_orphans: + # Drop the reviewed orphans that are still orphans right now. + current = find_orphans(read_data_table_rows(), list_data_db_tables(), timezone.now()) + dropped = drop_reviewed_orphans(reviewed, current) + + # List every table still marked unverified, from this run or any earlier one. + unverified = find_unverified(read_data_table_rows(), list_data_db_tables()) + + # Write the full report and print the summary. + report = build_report( + mode, + now, + entries, + plan.orphans, + dropped, + touched, + unverified, + plan.shared_groups, + ) + write_report(report_path, report) + if nothing_left_to_do(entries): + self.stdout.write("nothing to do") + for line in summary_lines(report, report_path): + self.stdout.write(line) diff --git a/api/management/commands/migrate_media_to_s3.py b/api/management/commands/migrate_media_to_s3.py new file mode 100644 index 00000000..a85bf0d7 --- /dev/null +++ b/api/management/commands/migrate_media_to_s3.py @@ -0,0 +1,144 @@ +"""Bring every stored file into S3 under its current, prefix-free name. + + python manage.py migrate_media_to_s3 # dry run + python manage.py migrate_media_to_s3 --execute # copy + rename + python manage.py migrate_media_to_s3 --source /code/files/public --execute + +Rows created on disk carry a ``files/public/`` or ``files/`` prefix that the S3 +keys no longer use (see ``normalize_media_name``). For each stored name this +puts the file at the normalised key in the storage its field names, taking it +from the old S3 key or, with ``--source``, from the old disk tree, then points +the row at the new name. Safe to re-run: names already in place are skipped. +Nothing is deleted, neither local files nor old S3 objects. +""" + +from collections import Counter +from pathlib import Path +from typing import Any, Iterator, List, Optional, Set, Tuple, Type + +import structlog +from django.apps import apps +from django.core.files import File +from django.core.files.storage import Storage +from django.core.management.base import BaseCommand, CommandError, CommandParser +from django.db import models + +from api.utils.storage import normalize_media_name + +logger = structlog.get_logger("dataspace.commands.migrate_media_to_s3") + + +def iter_file_fields() -> Iterator[Tuple[Type[models.Model], models.FileField]]: + """Yield every concrete file field across installed models.""" + for model in apps.get_models(): + for field in model._meta.get_fields(): + if isinstance(field, models.FileField): + yield model, field + + +def stored_names(model: Type[models.Model], field: models.FileField) -> List[str]: + """Distinct non-empty names stored in ``field``.""" + return list( + model._default_manager.exclude(**{field.name: ""}) + .exclude(**{f"{field.name}__isnull": True}) + .values_list(field.name, flat=True) + .distinct() + ) + + +def copy_to(storage: Storage, target: str, fh: Any) -> None: + """Save ``fh`` under exactly ``target``; the row will point at that name.""" + saved = storage.save(target, File(fh)) + if saved != target: + storage.delete(saved) + raise RuntimeError(f"storage renamed {target} to {saved}") + + +class Command(BaseCommand): + help = "Copy media into S3 under prefix-free names and point DB rows at them." + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument( + "--source", + help="Old MEDIA_ROOT that legacy names are relative to (e.g. /code/files/public)", + ) + parser.add_argument( + "--execute", action="store_true", help="Copy and rename; without it, only report." + ) + + def handle(self, *args: Any, **options: Any) -> None: + source: Optional[Path] = None + if options["source"]: + source = Path(options["source"]).resolve() + if not source.is_dir(): + raise CommandError(f"Source directory not found: {source}") + execute: bool = options["execute"] + + counts: Counter = Counter() + seen: Set[str] = set() + for model, field in iter_file_fields(): + storage = field.storage + for name in stored_names(model, field): + target = normalize_media_name(name) + seen.add(target) + try: + present = self._place(storage, name, target, source, execute, counts) + if present and target != name: + counts["rows_renamed"] += 1 + if execute: + model._default_manager.filter(**{field.name: name}).update( + **{field.name: target} + ) + except Exception as exc: + counts["failed"] += 1 + logger.error("media_migration_failed", name=name, error=str(exc)) + + if source: + counts["orphaned"] = sum( + 1 + for p in source.rglob("*") + # Disk files keep their legacy names; rows may already use the clean ones. + if p.is_file() + and normalize_media_name(p.relative_to(source).as_posix()) not in seen + ) + + summary = ", ".join(f"{key}={counts[key]}" for key in sorted(counts)) + logger.info("media_migration_summary", execute=execute, **dict(counts)) + self.stdout.write(f"{'Done' if execute else 'Dry run'}: {summary}") + if counts["failed"]: + raise CommandError(f"{counts['failed']} file(s) failed") + + def _place( + self, + storage: Storage, + name: str, + target: str, + source: Optional[Path], + execute: bool, + counts: Counter, + ) -> bool: + """Make sure ``target`` exists in ``storage``; return whether it does (or would).""" + if storage.exists(target): + counts["in_place"] += 1 + return True + + if target != name and storage.exists(name): + counts["moved_in_s3"] += 1 + if execute: + with storage.open(name, "rb") as fh: + copy_to(storage, target, fh) + return True + + if source is not None: + local = (source / name).resolve() + # Names come from the DB; never read outside the source tree. + if local.is_relative_to(source) and local.is_file(): + counts["copied_from_disk"] += 1 + if execute: + with local.open("rb") as fh: + copy_to(storage, target, fh) + return True + + counts["missing"] += 1 + logger.warning("media_source_missing", name=name) + return False diff --git a/api/management/commands/setup_dvc.py b/api/management/commands/setup_dvc.py deleted file mode 100644 index 22808721..00000000 --- a/api/management/commands/setup_dvc.py +++ /dev/null @@ -1,131 +0,0 @@ -import os -import subprocess -from typing import Any, Optional, cast - -import structlog -from django.conf import settings -from django.core.management.base import BaseCommand - -logger = structlog.getLogger(__name__) - - -class Command(BaseCommand): - help = "Set up DVC repository for dataset versioning" - - def handle(self, *args: Any, **options: Any) -> None: - try: - repo_path = settings.DVC_REPO_PATH - - # Create directory if needed - if not os.path.exists(repo_path): - os.makedirs(repo_path) - self.stdout.write(f"Created DVC repository directory at {repo_path}") - - # Initialize Git and DVC if not already done - if not os.path.exists(os.path.join(repo_path, ".git")): - subprocess.run(["git", "init"], cwd=repo_path, check=True) - self.stdout.write("Initialized Git repository") - - if not os.path.exists(os.path.join(repo_path, ".dvc")): - subprocess.run(["dvc", "init"], cwd=repo_path, check=True) - self.stdout.write("Initialized DVC repository") - - # Configure chunking for large files - subprocess.run( - ["dvc", "config", "cache.type", "hardlink,symlink"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["dvc", "config", "cache.shared", "group"], - cwd=repo_path, - check=True, - ) - - # Configure cache size limits to prevent excessive disk usage - subprocess.run( - ["dvc", "config", "cache.size_limit", "10G"], - cwd=repo_path, - check=True, - ) - - self.stdout.write("Configured DVC for large file handling") - - # Set up remote if configured - if ( - hasattr(settings, "DVC_REMOTE_NAME") - and hasattr(settings, "DVC_REMOTE_URL") - and settings.DVC_REMOTE_NAME - and settings.DVC_REMOTE_URL - ): - # Cast to str to satisfy mypy - remote_name = cast(str, settings.DVC_REMOTE_NAME) - remote_url = cast(str, settings.DVC_REMOTE_URL) - - subprocess.run( - [ - "dvc", - "remote", - "add", - remote_name, - remote_url, - ], - cwd=repo_path, - check=True, - ) - - # Set as default remote - subprocess.run( - ["dvc", "remote", "default", remote_name], - cwd=repo_path, - check=True, - ) - - self.stdout.write(f"Configured DVC remote: {remote_name}") - - # Set up Git user if not already configured - try: - # Check if Git user is configured - result = subprocess.run( - ["git", "config", "user.name"], - cwd=repo_path, - capture_output=True, - text=True, - ) - if not result.stdout.strip(): - subprocess.run( - ["git", "config", "user.name", "DataEx Bot"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "dataex@example.com"], - cwd=repo_path, - check=True, - ) - self.stdout.write("Configured Git user for DVC repository") - except subprocess.CalledProcessError: - # If checking fails, set the user anyway - subprocess.run( - ["git", "config", "user.name", "DataEx Bot"], - cwd=repo_path, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "dataex@example.com"], - cwd=repo_path, - check=True, - ) - self.stdout.write("Configured Git user for DVC repository") - - self.stdout.write(self.style.SUCCESS("DVC repository set up successfully")) - - except Exception as e: - import traceback - - logger.error( - f"Failed to set up DVC repository: {str(e)}, Traceback: {traceback.format_exc()}" - ) - self.stdout.write( - self.style.ERROR(f"Failed to set up DVC repository: {str(e)}") - ) diff --git a/api/management/commands/sync_search_mappings.py b/api/management/commands/sync_search_mappings.py index 73552df6..b2351c3c 100644 --- a/api/management/commands/sync_search_mappings.py +++ b/api/management/commands/sync_search_mappings.py @@ -18,9 +18,17 @@ background on the Elasticsearch side (no rebuild, no downtime; until it finishes, searches on the new sub-fields simply match fewer documents). -It skips indexes that do not exist yet (those are created with the full mapping -on first use) and never fails the process: problems are logged and reported, -so a search outage cannot stop the backend from starting. +When top-level fields were added, their values come from the database, which +Elasticsearch cannot compute itself. Existing documents are then updated in +place with just those fields (a partial update per record, not a reindex), so +for example search results for old datasets get ``url_slug`` right away. + +A registered document whose index does not exist yet (a new search type, such +as events) gets its index created, empty, with the full mapping and analysers. +Without that, the first published item would make Elasticsearch create the +index itself with guessed field types, and aggregations and sorting on it would +fail. The command never fails the process: problems are logged and reported, so +a search outage cannot stop the backend from starting. python manage.py sync_search_mappings # apply python manage.py sync_search_mappings --dry-run # report only @@ -30,11 +38,12 @@ from __future__ import annotations -from typing import Any, Dict, List +from typing import Any, Dict, Iterator, List import structlog from django.core.management.base import BaseCommand from django_elasticsearch_dsl.registries import registry +from elasticsearch.helpers import bulk logger = structlog.get_logger("dataspace.search_mappings") @@ -64,6 +73,39 @@ def missing_subfields(existing: Dict[str, Any], desired: Dict[str, Any]) -> Dict return updates +def refill_fields(document: Any, es: Any, index_name: str, names: List[str]) -> int: + """Write the current values of ``names`` into every existing document. + + Only the given fields are sent (a partial update), computed the same way + the document computes them when a record is saved. Records that are not in + the index (for example ones the document never indexes) are skipped. + Returns the number of documents updated. + """ + doc = document() + preparers = [(name, fn) for name, _field, fn in doc._prepared_fields if name in names] + if not preparers: + return 0 + + def actions() -> Iterator[Dict[str, Any]]: + for instance in doc.get_queryset().iterator(chunk_size=500): + try: + values = {name: fn(instance) for name, fn in preparers} + except Exception as exc: # one bad record must not stop the rest + logger.warning( + "search_refill_skipped", index=index_name, pk=str(instance.pk), error=str(exc) + ) + continue + yield { + "_op_type": "update", + "_index": index_name, + "_id": instance.pk, + "doc": values, + } + + updated, _failed = bulk(es, actions(), raise_on_error=False, stats_only=True) + return int(updated) + + class Command(BaseCommand): help = "Add fields and sub-fields that search documents define but their live indexes lack." @@ -77,7 +119,12 @@ def handle(self, *args: Any, **options: Any) -> None: try: es = document._get_connection() if not es.indices.exists(index=index_name): - self.stdout.write(f"{index_name}: no index yet, skipped") + if dry_run: + self.stdout.write(f"{index_name}: would create the index") + continue + document._index.create(using=es) + self.stdout.write(self.style.SUCCESS(f"{index_name}: created the index")) + logger.info("search_index_created", index=index_name) continue live = es.indices.get_mapping(index=index_name) existing: Dict[str, Any] = {} @@ -97,8 +144,9 @@ def handle(self, *args: Any, **options: Any) -> None: added += [f"{name}.{sub}" for sub in sorted(spec["fields"]) if sub not in have] names = ", ".join(added) if dry_run: - refill = " and refill existing documents" if subfield_updates else "" - self.stdout.write(f"{index_name}: would add {names}{refill}") + self.stdout.write( + f"{index_name}: would add {names} and refill existing documents" + ) continue es.indices.put_mapping( @@ -112,6 +160,9 @@ def handle(self, *args: Any, **options: Any) -> None: index=index_name, conflicts="proceed", wait_for_completion=False ) message += f"; refilling existing documents (task {task.get('task')})" + if fields_to_add: + count = refill_fields(document, es, index_name, sorted(fields_to_add)) + message += f"; filled new fields in {count} existing documents" self.stdout.write(self.style.SUCCESS(message)) logger.info("search_mapping_fields_added", index=index_name, fields=added) except Exception as exc: # never block start-up on search problems diff --git a/api/management/commands/update_dataindex.py b/api/management/commands/update_dataindex.py index af1190a4..b9647f77 100644 --- a/api/management/commands/update_dataindex.py +++ b/api/management/commands/update_dataindex.py @@ -4,7 +4,6 @@ import structlog from django.core.management.base import BaseCommand, CommandParser -from django.db import transaction from api.models.Resource import Resource, ResourceDataTable from api.utils.data_indexing import INDEXED_FORMATS, index_resource_data @@ -125,9 +124,9 @@ def handle(self, *args: Any, **options: Dict[str, Any]) -> None: f"[{i}/{total_resources}] Processing resource {resource.id} - {resource.name}" ) - # Update dataindex for the resource - with transaction.atomic(): - data_table = index_resource_data(resource) + # Not inside a transaction: the flow commits its own switch and refuses + # to run inside an outer one (a failed resource keeps its old data). + data_table = index_resource_data(resource) if data_table: success_count += 1 diff --git a/api/managers/dvc_manager.py b/api/managers/dvc_manager.py deleted file mode 100644 index e9c98195..00000000 --- a/api/managers/dvc_manager.py +++ /dev/null @@ -1,132 +0,0 @@ -import subprocess -from pathlib import Path -from typing import List, Union - -import structlog - -logger = structlog.getLogger(__name__) - - -class DVCManager: - def __init__(self, repo_path: Union[str, Path]) -> None: - self.repo_path = Path(repo_path) - - def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: - try: - return subprocess.run( - command, - cwd=str(self.repo_path), - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - # Log the error with details - logger.error(f"Command failed: {' '.join(command)}, Error: {e.stderr}") - # You can choose to re-raise or handle differently - raise - - def track_resource(self, file_path: Union[str, Path], chunked: bool = False) -> str: - """Add a resource file to DVC tracking with optional chunking for large files""" - rel_path = Path(file_path).relative_to(self.repo_path) - cmd = ["dvc", "add"] - if chunked: - cmd.append("--chunked") # Enables chunking for large files - cmd.append(str(rel_path)) - self._run_command(cmd) - return str(rel_path) + ".dvc" - - def commit_version(self, dvc_file: str, message: str) -> None: - """Commit the DVC file to git""" - self._run_command(["git", "add", dvc_file]) - self._run_command(["git", "commit", "-m", message]) - - def tag_version(self, tag_name: str) -> None: - """Add a git tag for this version""" - self._run_command(["git", "tag", tag_name]) - - def push_to_remote(self) -> None: - """Push data to DVC remote and metadata to git""" - self._run_command(["dvc", "push"]) - self._run_command(["git", "push", "--follow-tags"]) - - def get_version(self) -> str: - """Get the current version from DVC""" - result = self._run_command(["dvc", "version"]) - return str(result.stdout.strip()) - - def get_remote(self) -> str: - """Get the current remote from DVC""" - result = self._run_command(["dvc", "remote", "list"]) - return str(result.stdout.strip()) - - def setup_remote(self, remote_name: str, remote_url: str) -> None: - """Configure a DVC remote for storing data""" - self._run_command(["dvc", "remote", "add", remote_name, remote_url]) - self._run_command(["dvc", "remote", "default", remote_name]) - - def verify_file(self, file_path: Union[str, Path]) -> bool: - """Verify file integrity using DVC checksums""" - rel_path = Path(file_path).relative_to(self.repo_path) - try: - result = self._run_command(["dvc", "status", str(rel_path)]) - return "up to date" in str(result.stdout) - except subprocess.CalledProcessError: - return False - - def rollback_to_version( - self, file_path: Union[str, Path], version_tag: str - ) -> None: - """Roll back a file to a specific version using DVC and Git""" - rel_path = Path(file_path).relative_to(self.repo_path) - dvc_file = str(rel_path) + ".dvc" - - # Checkout the specific version of the DVC file - self._run_command(["git", "checkout", version_tag, "--", dvc_file]) - - # Pull the data for that version - self._run_command(["dvc", "checkout", str(rel_path)]) - - def lock_resource(self, file_path: Union[str, Path]) -> None: - """Lock a resource to prevent concurrent modifications""" - rel_path = Path(file_path).relative_to(self.repo_path) - self._run_command(["dvc", "lock", str(rel_path)]) - - def unlock_resource(self, file_path: Union[str, Path]) -> None: - """Unlock a previously locked resource""" - rel_path = Path(file_path).relative_to(self.repo_path) - self._run_command(["dvc", "unlock", str(rel_path)]) - - def add_metric(self, metric_file: Union[str, Path], metric_name: str) -> None: - """Add a file as a DVC metric for tracking""" - rel_path = Path(metric_file).relative_to(self.repo_path) - self._run_command( - ["dvc", "metrics", "add", str(rel_path), "--name", metric_name] - ) - - def show_metrics(self) -> str: - """Show all tracked metrics""" - result = self._run_command(["dvc", "metrics", "show"]) - return str(result.stdout.strip()) - - def gc_cache(self, force: bool = False) -> None: - """Clean up unused cache to save disk space""" - cmd = ["dvc", "gc"] - if force: - cmd.append("-f") - self._run_command(cmd) - - def configure(self, section: str, option: str, value: str) -> None: - """Configure DVC settings""" - self._run_command(["dvc", "config", f"{section}.{option}", value]) - - def has_changes(self, file_path: Union[str, Path]) -> bool: - """Check if a file has uncommitted changes""" - rel_path = Path(file_path).relative_to(self.repo_path) - try: - result = self._run_command(["dvc", "status", str(rel_path)]) - # If there are changes, the output will contain the file path - return str(rel_path) in str(result.stdout) - except subprocess.CalledProcessError: - # If the command fails, assume there are changes - return True diff --git a/api/migrations/0054_add_source_identifier_metadata.py b/api/migrations/0054_add_source_identifier_metadata.py new file mode 100644 index 00000000..f39e0d8c --- /dev/null +++ b/api/migrations/0054_add_source_identifier_metadata.py @@ -0,0 +1,48 @@ +"""Add the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers (DataSpace-Scrapers) fill it on every dataset they upload with +dataspace-scrapers//, and read it back to recognise their own +datasets, so a dataset is never created twice and their upload records can be rebuilt +from DataSpace. People never fill it in. Platform imports (Kaggle, Hugging Face, GitHub) +also fill it, with the platform's own ID, because the label matches the import's +"source identifier" field; only values starting with dataspace-scrapers/ are a scraper's. + +Creates the item if it is missing and switches it back on if an admin switched it off. +Reversing it deletes nothing, since datasets may already carry values for it. +""" + +from django.db import migrations + +LABEL = "Source identifier" + + +def add_source_identifier(apps, schema_editor): + Metadata = apps.get_model("api", "Metadata") + item, created = Metadata.objects.get_or_create( + label=LABEL, + model="DATASET", + defaults={ + "urn": "dataspace-scrapers:source-identifier", + "data_type": "STRING", + "type": "OPTIONAL", + "data_standard": "NA", + "options": None, + "validator": [], + "enabled": True, + "filterable": False, + }, + ) + if not created and not item.enabled: + item.enabled = True + item.save(update_fields=["enabled"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0053_alter_collaborative_summary"), + ] + + operations = [ + migrations.RunPython(add_source_identifier, migrations.RunPython.noop), + ] diff --git a/api/migrations/0055_event.py b/api/migrations/0055_event.py new file mode 100644 index 00000000..fee78c6b --- /dev/null +++ b/api/migrations/0055_event.py @@ -0,0 +1,76 @@ +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0054_add_source_identifier_metadata"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Event", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + ("name", models.CharField(blank=True, max_length=300)), + ("public_id", models.CharField(editable=False, max_length=12, unique=True)), + ("location", models.CharField(blank=True, max_length=300)), + ("start_date", models.DateField(blank=True, null=True)), + ("end_date", models.DateField(blank=True, null=True)), + ("luma_embed", models.JSONField(blank=True, default=dict)), + ( + "status", + models.CharField( + choices=[("DRAFT", "Draft"), ("PUBLISHED", "Published")], + default="DRAFT", + max_length=50, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "organization", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="events", + to="api.organization", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="events", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "Event", + "verbose_name_plural": "Events", + "db_table": "event", + "ordering": ["-modified"], + "indexes": [ + models.Index( + fields=["organization", "-modified"], name="event_org_modified_idx" + ), + models.Index(fields=["user", "-modified"], name="event_user_modified_idx"), + models.Index(fields=["status"], name="event_status_idx"), + ], + }, + ), + ] diff --git a/api/migrations/0056_update_license_choices.py b/api/migrations/0056_update_license_choices.py new file mode 100644 index 00000000..bd87449d --- /dev/null +++ b/api/migrations/0056_update_license_choices.py @@ -0,0 +1,285 @@ +# Generated by Django 5.0.4 on 2026-10-31 18:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0055_event"), + ] + + operations = [ + migrations.AlterField( + model_name="dataset", + name="license", + field=models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License – India"), + ("CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License 1.0"), + ("CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal"), + ("CC_BY_1_0", "Creative Commons Attribution 1.0 Generic"), + ("CC_BY_2_0", "Creative Commons Attribution 2.0 Generic"), + ("CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia"), + ("CC_BY_2_5", "Creative Commons Attribution 2.5 Generic"), + ("CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia"), + ("CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria"), + ("CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany"), + ("CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO"), + ("CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands"), + ("CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States"), + ("CC_BY_3_0", "Creative Commons Attribution 3.0 Unported"), + ("CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic"), + ("CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic"), + ("CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic"), + ("CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany"), + ("CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported"), + ( + "CC_BY_ND_4_0", + "Creative Commons Attribution No Derivatives 4.0 International", + ), + ("CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic"), + ("CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic"), + ("CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic"), + ("CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany"), + ("CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO"), + ("CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported"), + ( + "CC_BY_NC_4_0", + "Creative Commons Attribution Non Commercial 4.0 International", + ), + ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ), + ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ), + ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ), + ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ), + ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ), + ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ), + ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ), + ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ), + ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ), + ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ), + ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ), + ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ), + ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ), + ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ), + ("CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic"), + ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ), + ("CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic"), + ("CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan"), + ("CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic"), + ("CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria"), + ("CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany"), + ("CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported"), + ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ), + ("CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO"), + ("CC_PDDC", "Creative Commons Public Domain Dedication and Certification"), + ("CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic"), + ("CC0_1_0", "Creative Commons Zero v1.0 Universal"), + ("PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + migrations.AlterField( + model_name="publication", + name="license", + field=models.CharField( + choices=[ + ("GOVERNMENT_OPEN_DATA_LICENSE", "Government Open Data License – India"), + ("CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0"), + ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ), + ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ), + ("OPEN_DATABASE_LICENSE", "Open Database License 1.0"), + ("CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal"), + ("CC_BY_1_0", "Creative Commons Attribution 1.0 Generic"), + ("CC_BY_2_0", "Creative Commons Attribution 2.0 Generic"), + ("CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia"), + ("CC_BY_2_5", "Creative Commons Attribution 2.5 Generic"), + ("CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia"), + ("CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria"), + ("CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany"), + ("CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO"), + ("CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands"), + ("CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States"), + ("CC_BY_3_0", "Creative Commons Attribution 3.0 Unported"), + ("CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic"), + ("CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic"), + ("CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic"), + ("CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany"), + ("CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported"), + ( + "CC_BY_ND_4_0", + "Creative Commons Attribution No Derivatives 4.0 International", + ), + ("CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic"), + ("CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic"), + ("CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic"), + ("CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany"), + ("CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO"), + ("CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported"), + ( + "CC_BY_NC_4_0", + "Creative Commons Attribution Non Commercial 4.0 International", + ), + ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ), + ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ), + ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ), + ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ), + ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ), + ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ), + ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ), + ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ), + ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ), + ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ), + ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ), + ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ), + ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ), + ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ), + ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ), + ("CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic"), + ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ), + ("CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic"), + ("CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan"), + ("CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic"), + ("CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria"), + ("CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany"), + ("CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported"), + ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ), + ("CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO"), + ("CC_PDDC", "Creative Commons Public Domain Dedication and Certification"), + ("CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic"), + ("CC0_1_0", "Creative Commons Zero v1.0 Universal"), + ("PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0"), + ], + default="CC_BY_4_0_ATTRIBUTION", + max_length=50, + ), + ), + ] diff --git a/api/migrations/0057_repair_event_search_index.py b/api/migrations/0057_repair_event_search_index.py new file mode 100644 index 00000000..98c333ad --- /dev/null +++ b/api/migrations/0057_repair_event_search_index.py @@ -0,0 +1,47 @@ +"""One-time repair of the ``event`` search index. Touches Elasticsearch only. + +The first deploy with event search populated the index before it was created +(``api.Event`` was missing from the create list in scripts/ci-deploy.sh, fixed +since), so Elasticsearch created it with guessed field types and every event +search failed. If the live index has such a guessed mapping (``status`` not a +keyword), drop it: the deploy's release step runs migrations first, then +creates each search index and populates it, so the next steps rebuild it with +the real mapping and refill it from the database. + +Does nothing when the index is missing or already correct, and never fails the +migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def drop_guessed_event_index(apps, schema_editor): + try: + from elasticsearch_dsl.connections import connections + + index_name = settings.ELASTICSEARCH_INDEX_NAMES["search.documents.event_document"] + es = connections.get_connection().options(request_timeout=10) + if not es.indices.exists(index=index_name): + return + for concrete, body in es.indices.get_mapping(index=index_name).items(): + status = body.get("mappings", {}).get("properties", {}).get("status", {}) + if status.get("type") != "keyword": + es.indices.delete(index=concrete) + logger.info("guessed_event_index_dropped", index=concrete) + except Exception as exc: # never block a deploy on search problems + logger.error("event_index_repair_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0056_update_license_choices"), + ] + + operations = [ + migrations.RunPython(drop_guessed_event_index, migrations.RunPython.noop), + ] diff --git a/api/migrations/0058_file_field_storages.py b/api/migrations/0058_file_field_storages.py new file mode 100644 index 00000000..26ed586d --- /dev/null +++ b/api/migrations/0058_file_field_storages.py @@ -0,0 +1,108 @@ +# Generated by Django 5.0.4 on 2026-10-12 20:25 + +from django.db import migrations, models + +import api.utils.file_paths +import api.utils.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0057_repair_event_search_index"), + ] + + operations = [ + migrations.AlterField( + model_name="catalog", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._catalog_directory_path, + ), + ), + migrations.AlterField( + model_name="collaborative", + name="cover_image", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + migrations.AlterField( + model_name="collaborative", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + migrations.AlterField( + model_name="dataspace", + name="logo", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._dataspace_directory_path, + ), + ), + migrations.AlterField( + model_name="organization", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._organization_directory_path, + ), + ), + migrations.AlterField( + model_name="publicationblock", + name="file", + field=models.FileField( + blank=True, + max_length=300, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._publication_block_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcechartimage", + name="image", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.private_storage, + upload_to=api.utils.file_paths._chart_image_directory_path, + ), + ), + migrations.AlterField( + model_name="resourcefiledetails", + name="file", + field=models.FileField( + max_length=300, storage=api.utils.storage.private_storage, upload_to="resources/" + ), + ), + migrations.AlterField( + model_name="usecase", + name="logo", + field=models.ImageField( + blank=True, + max_length=300, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._use_case_directory_path, + ), + ), + ] diff --git a/api/migrations/0059_event_search_cover_url.py b/api/migrations/0059_event_search_cover_url.py new file mode 100644 index 00000000..fc051a97 --- /dev/null +++ b/api/migrations/0059_event_search_cover_url.py @@ -0,0 +1,40 @@ +"""Add ``cover_url`` to a live ``event`` search index. Touches Elasticsearch only. + +The deploy's release step refills every index right after ``migrate`` and +before ``sync_search_mappings`` runs. Without this step Elasticsearch would +guess the type of the new ``cover_url`` field on the first refilled document. +Does nothing when the index is missing (the release step then creates it with +the full mapping), and never fails the migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def add_cover_url_field(apps, schema_editor): + try: + from elasticsearch_dsl.connections import connections + + index_name = settings.ELASTICSEARCH_INDEX_NAMES["search.documents.event_document"] + es = connections.get_connection().options(request_timeout=10) + if es.indices.exists(index=index_name): + es.indices.put_mapping( + index=index_name, + properties={"cover_url": {"type": "keyword", "index": False}}, + ) + except Exception as exc: # never block a deploy on search problems + logger.error("event_cover_mapping_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0058_file_field_storages"), + ] + + operations = [ + migrations.RunPython(add_cover_url_field, migrations.RunPython.noop), + ] diff --git a/api/migrations/0060_public_ids.py b/api/migrations/0060_public_ids.py new file mode 100644 index 00000000..587bb998 --- /dev/null +++ b/api/migrations/0060_public_ids.py @@ -0,0 +1,53 @@ +from django.db import migrations, models + +MODELS = ["aimodel", "collaborative", "dataset", "publication", "usecase"] + + +class Migration(migrations.Migration): + """Add public_id (nullable for now) to every public module. + + Also drops the unique rule on dataset, publication and use case slugs and + on use case / collaborative titles: links use public_id, so titles may + repeat. Collaborative slugs stay unique (the slug is also the subdomain). + Filled for existing rows in 0061, made unique and required in 0062. + """ + + dependencies = [ + ("api", "0059_event_search_cover_url"), + ] + + operations = [ + *[ + migrations.AddField( + model_name=model, + name="public_id", + field=models.CharField(editable=False, max_length=12, null=True), + ) + for model in MODELS + ], + migrations.AlterField( + model_name="collaborative", + name="title", + field=models.CharField(blank=True, max_length=200, null=True), + ), + migrations.AlterField( + model_name="dataset", + name="slug", + field=models.SlugField(max_length=255), + ), + migrations.AlterField( + model_name="publication", + name="slug", + field=models.SlugField(max_length=255), + ), + migrations.AlterField( + model_name="usecase", + name="slug", + field=models.SlugField(blank=True, max_length=75, null=True), + ), + migrations.AlterField( + model_name="usecase", + name="title", + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/api/migrations/0061_fill_public_ids.py b/api/migrations/0061_fill_public_ids.py new file mode 100644 index 00000000..2592705a --- /dev/null +++ b/api/migrations/0061_fill_public_ids.py @@ -0,0 +1,20 @@ +from django.db import migrations + +from api.utils.public_links import fill_missing_public_ids + +MODELS = ["AIModel", "Collaborative", "Dataset", "Publication", "UseCase"] + + +def fill(apps, schema_editor): + for name in MODELS: + fill_missing_public_ids(apps.get_model("api", name)) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0060_public_ids"), + ] + + operations = [ + migrations.RunPython(fill, migrations.RunPython.noop), + ] diff --git a/api/migrations/0062_public_ids_unique.py b/api/migrations/0062_public_ids_unique.py new file mode 100644 index 00000000..137d124b --- /dev/null +++ b/api/migrations/0062_public_ids_unique.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + +MODELS = ["aimodel", "collaborative", "dataset", "publication", "usecase"] + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0061_fill_public_ids"), + ] + + operations = [ + migrations.AlterField( + model_name=model, + name="public_id", + field=models.CharField(editable=False, max_length=12, unique=True), + ) + for model in MODELS + ] diff --git a/api/migrations/0063_resource_slug_not_unique.py b/api/migrations/0063_resource_slug_not_unique.py new file mode 100644 index 00000000..4b8169be --- /dev/null +++ b/api/migrations/0063_resource_slug_not_unique.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Resource slugs no longer need to be unique (#229). + + Nothing looks a resource up by slug, and the unique rule made two uploads + of the same file name at the same moment fail. + """ + + dependencies = [ + ("api", "0062_public_ids_unique"), + ] + + operations = [ + migrations.AlterField( + model_name="resource", + name="slug", + field=models.SlugField(max_length=255), + ), + ] diff --git a/api/migrations/0064_rebuild_long_text_search_indexes.py b/api/migrations/0064_rebuild_long_text_search_indexes.py new file mode 100644 index 00000000..818498ae --- /dev/null +++ b/api/migrations/0064_rebuild_long_text_search_indexes.py @@ -0,0 +1,71 @@ +"""One-time rebuild of search indexes that still have a single-token text copy. + +Long text fields (use case and collaborative summaries, descriptions, bios) had +a ``raw`` sub-field with the ``keyword`` analyzer, which turns the whole value +into one token. Elasticsearch refuses tokens over 32,766 bytes, so a long +summary failed to index: saving a published item returned an error (#266) and +every deploy failed at ``search_index --populate``. The documents no longer +define those sub-fields, but an existing index keeps its old mapping. + +For each search index whose live mapping still has such a sub-field, drop it +and create it again empty with the current mapping. The deploy's release step +runs migrations first and then populates every index from the database, so +search is refilled right after. Nothing in the database changes. + +Does nothing when an index is missing or already clean, and never fails the +migration: problems are only logged. +""" + +import structlog +from django.conf import settings +from django.db import migrations + +logger = structlog.get_logger(__name__) + + +def _has_single_token_text(properties): + for field in properties.values(): + for sub in field.get("fields", {}).values(): + if sub.get("type") == "text" and sub.get("analyzer") == "keyword": + return True + if _has_single_token_text(field.get("properties", {})): + return True + return False + + +def rebuild_long_text_indexes(apps, schema_editor): + try: + from django_elasticsearch_dsl.registries import registry + from elasticsearch_dsl.connections import connections + + es = connections.get_connection().options(request_timeout=30) + indices = {index._name: index for index in registry.get_indices()} + for name in set(settings.ELASTICSEARCH_INDEX_NAMES.values()): + if not es.indices.exists(index=name): + continue + mappings = es.indices.get_mapping(index=name) + if not any( + _has_single_token_text(body.get("mappings", {}).get("properties", {})) + for body in mappings.values() + ): + continue + for concrete in mappings: + es.indices.delete(index=concrete) + # Create it again at once so a write from the running server cannot + # make Elasticsearch create it with guessed field types. + if name in indices: + indices[name].create() + logger.info("long_text_search_index_rebuilt", index=name) + except Exception as exc: # never block a deploy on search problems + logger.error("long_text_search_index_rebuild_failed", error=str(exc)) + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0063_resource_slug_not_unique"), + ] + + operations = [ + migrations.RunPython(rebuild_long_text_indexes, migrations.RunPython.noop), + ] diff --git a/api/migrations/0065_merge_search_hotfix.py b/api/migrations/0065_merge_search_hotfix.py new file mode 100644 index 00000000..8d27ddc8 --- /dev/null +++ b/api/migrations/0065_merge_search_hotfix.py @@ -0,0 +1,18 @@ +"""Joins the main hotfix migration 0054_rebuild_long_text_search_indexes with dev. + +The hotfix went to main on its own (main was at 0053), so both branches have a +migration after their own parent. This merge keeps one migration leaf. Both +rebuild migrations do the same thing and do nothing once indexes are clean. +""" + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0064_rebuild_long_text_search_indexes"), + ("api", "0054_rebuild_long_text_search_indexes"), + ] + + operations = [] diff --git a/api/models/AIModel.py b/api/models/AIModel.py index 9646e627..25239028 100644 --- a/api/models/AIModel.py +++ b/api/models/AIModel.py @@ -14,6 +14,7 @@ HFModelClass, PromptDomain, ) +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id User = get_user_model() @@ -27,6 +28,7 @@ class AIModel(models.Model): # Basic Information name = models.CharField(max_length=255, help_text="Model name or identifier") display_name = models.CharField(max_length=255, help_text="Human-readable name") + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) version = models.CharField(max_length=50, blank=True, help_text="Model version") description = models.TextField(help_text="Description of the model's capabilities") @@ -157,6 +159,13 @@ class Meta: def __str__(self): return f"{self.display_name} ({self.provider})" + def save(self, *args, **kwargs): + save_with_public_id(self, lambda: super(AIModel, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + return build_url_slug(self.display_name or self.name, self.public_id) + @property def tags_indexing(self) -> list[str]: """Tags for indexing. diff --git a/api/models/Catalog.py b/api/models/Catalog.py index 101bb3d7..799a82f6 100644 --- a/api/models/Catalog.py +++ b/api/models/Catalog.py @@ -6,12 +6,15 @@ from django.utils.translation import gettext_lazy as _ from api.utils.file_paths import _catalog_directory_path +from api.utils.storage import public_storage class Catalog(models.Model): name = models.CharField(max_length=255) datasets = models.ManyToManyField("api.Dataset", related_name="catalogs") - logo = models.ImageField(upload_to=_catalog_directory_path, blank=True, null=True) + logo = models.ImageField( + upload_to=_catalog_directory_path, storage=public_storage, blank=True, null=True + ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) diff --git a/api/models/Collaborative.py b/api/models/Collaborative.py index 3b9a38a7..99020d71 100644 --- a/api/models/Collaborative.py +++ b/api/models/Collaborative.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Any, cast from django.core.validators import RegexValidator -from django.db import models +from django.db import IntegrityError, models, transaction from django.utils.text import slugify if TYPE_CHECKING: @@ -10,9 +10,15 @@ from api.models.Organization import Organization from authorization.models import User -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.file_paths import _use_case_directory_path +from api.utils.public_links import ( + PUBLIC_ID_LENGTH, + build_url_slug, + generate_public_id, + save_with_public_id, +) +from api.utils.storage import public_storage slug_validator = RegexValidator( regex=r"^[a-z0-9]+(?:-[a-z0-9]+)*$", @@ -20,23 +26,38 @@ ) +def _slug_with_suffix(base: str) -> str: + """``base`` plus 4 random characters, kept within the 75-character limit.""" + return f"{base[:70].rstrip('-')}-{generate_public_id()[:4]}" + + class Collaborative(models.Model): id = models.AutoField(primary_key=True) - title = models.CharField(max_length=200, unique=True, blank=True, null=True) + title = models.CharField(max_length=200, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) cover_image = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) + # Unique: the slug is also the collaborative's subdomain. Page links use public_id. slug = models.SlugField( max_length=75, null=True, blank=True, unique=True, validators=[slug_validator] ) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( "api.Organization", on_delete=models.CASCADE, null=True, blank=True @@ -68,19 +89,34 @@ class Collaborative(models.Model): platform_url = models.URLField(blank=True, null=True) def save(self, *args: Any, **kwargs: Any) -> None: + base = "" if self.title and not self.slug: - self.slug = slugify(cast(str, self.title)) - base_title = self.title - - def disambiguate(attempt: int) -> None: - self.title = f"{base_title} ({attempt})" - self.slug = slugify(cast(str, self.title)) + base = slugify(cast(str, self.title))[:75].strip("-") + if base: + taken = Collaborative.objects.filter(slug=base).exclude(pk=self.pk).exists() + self.slug = _slug_with_suffix(base) if taken else base def do_save() -> None: - self.full_clean() + # public_id clashes are caught by the database and retried, not here. + self.full_clean(exclude=["public_id"]) super(Collaborative, self).save(*args, **kwargs) - retry_on_slug_collision(do_save, disambiguate) + # Titles may repeat but slugs may not. A slug made from the title that + # another save took in the meantime gets a new suffix; a slug the owner + # typed is never changed (full_clean refuses it if taken). + for attempt in range(3): + try: + with transaction.atomic(): + save_with_public_id(self, do_save) + return + except IntegrityError as exc: + if not base or "slug" not in str(exc) or attempt == 2: + raise + self.slug = _slug_with_suffix(base) + + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_collaborative(self): diff --git a/api/models/DataSpace.py b/api/models/DataSpace.py index 7430065f..1cb9a8fc 100644 --- a/api/models/DataSpace.py +++ b/api/models/DataSpace.py @@ -4,12 +4,15 @@ from django.utils.text import slugify from api.utils.file_paths import _dataspace_directory_path +from api.utils.storage import public_storage class DataSpace(models.Model): name = models.CharField(max_length=200) description = models.CharField(max_length=1000) - logo = models.ImageField(upload_to=_dataspace_directory_path, blank=True, null=True) + logo = models.ImageField( + upload_to=_dataspace_directory_path, storage=public_storage, blank=True, null=True + ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) diff --git a/api/models/Dataset.py b/api/models/Dataset.py index 296f509c..f3d3386d 100644 --- a/api/models/Dataset.py +++ b/api/models/Dataset.py @@ -5,13 +5,13 @@ from django.db.models import Sum from django.utils.text import slugify -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import ( DatasetAccessType, DatasetLicense, DatasetStatus, DatasetType, ) +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id if TYPE_CHECKING: from api.models.DataSpace import DataSpace @@ -36,7 +36,10 @@ class Dataset(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) title = models.CharField(max_length=300, unique=False, blank=True) description = models.CharField(max_length=1000, unique=False, blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Kept for older readers (SDK, search); no longer unique. Links use + # public_id, see api/utils/public_links.py. + slug = models.SlugField(max_length=255) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) organization = models.ForeignKey( "api.Organization", on_delete=models.SET_NULL, @@ -84,15 +87,13 @@ class Dataset(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - self.slug = slugify(self.title) - base_slug = self.slug + self.slug = slugify(self.title)[:255] + save_with_public_id(self, lambda: super(Dataset, self).save(*args, **kwargs)) - def disambiguate(attempt: int) -> None: - self.slug = f"{base_slug}-{attempt}" - - retry_on_slug_collision( - lambda: super(Dataset, self).save(*args, **kwargs), disambiguate - ) + @property + def url_slug(self) -> str: + """Link part for the dataset page, e.g. ``city-air-quality-index-k3x9m2qa7b1z``.""" + return build_url_slug(self.title, self.public_id) @property def tags_indexing(self) -> list[str]: diff --git a/api/models/Event.py b/api/models/Event.py new file mode 100644 index 00000000..0b6408fd --- /dev/null +++ b/api/models/Event.py @@ -0,0 +1,74 @@ +import uuid +from typing import Any + +from django.db import models + +from api.utils.enums import EventStatus +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id + + +class Event(models.Model): + """A Luma-backed event owned by an organization or an individual user. + + Peer to Publication: a UUID entity with a draft/publish status, linked by + ``public_id``. The embed is stored as whitelisted iframe attributes, never + as raw HTML. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=300, blank=True) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) + location = models.CharField(max_length=300, blank=True) + start_date = models.DateField(null=True, blank=True) + end_date = models.DateField(null=True, blank=True) + luma_embed = models.JSONField(default=dict, blank=True) + status = models.CharField( + max_length=50, + default=EventStatus.DRAFT, + choices=EventStatus.choices, + ) + + organization = models.ForeignKey( + "api.Organization", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="events", + ) + user = models.ForeignKey( + "authorization.User", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="events", + ) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def save(self, *args: Any, **kwargs: Any) -> None: + save_with_public_id(self, lambda: super(Event, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + """Public link part: title words plus public_id, e.g. ``data-meetup-k3x9m2qa7b1z``.""" + return build_url_slug(self.name, self.public_id) + + @property + def is_individual_event(self) -> bool: + """True when this event is owned by an individual, not an organization.""" + return self.organization is None and self.user is not None + + def __str__(self) -> str: + return self.name or "Untitled event" + + class Meta: + verbose_name = "Event" + verbose_name_plural = "Events" + db_table = "event" + ordering = ["-modified"] + indexes = [ + models.Index(fields=["organization", "-modified"], name="event_org_modified_idx"), + models.Index(fields=["user", "-modified"], name="event_user_modified_idx"), + models.Index(fields=["status"], name="event_status_idx"), + ] diff --git a/api/models/Organization.py b/api/models/Organization.py index 977f041a..96131f2d 100644 --- a/api/models/Organization.py +++ b/api/models/Organization.py @@ -5,21 +5,24 @@ from api.utils.enums import OrganizationTypes from api.utils.file_paths import _organization_directory_path +from api.utils.storage import public_storage class Organization(models.Model): name = models.CharField(max_length=200) description = models.CharField(max_length=1000) logo = models.ImageField( - upload_to=_organization_directory_path, blank=True, null=True, max_length=300 + upload_to=_organization_directory_path, + storage=public_storage, + blank=True, + null=True, + max_length=300, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) homepage = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - organization_types = models.CharField( - max_length=100, choices=OrganizationTypes.choices - ) + organization_types = models.CharField(max_length=100, choices=OrganizationTypes.choices) parent = models.ForeignKey( "api.Organization", unique=False, diff --git a/api/models/Publication.py b/api/models/Publication.py index 2acd2c5d..f1aa1a94 100644 --- a/api/models/Publication.py +++ b/api/models/Publication.py @@ -7,6 +7,8 @@ from api.utils.enums import DatasetLicense, PublicationBlockType, PublicationStatus from api.utils.file_paths import _publication_block_directory_path +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id +from api.utils.storage import public_storage if TYPE_CHECKING: from api.models.Organization import Organization @@ -27,7 +29,9 @@ class Publication(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) title = models.CharField(max_length=300, unique=False, blank=True) description = models.TextField(blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Kept for older readers (SDK); no longer unique. Links use public_id (see url_slug). + slug = models.SlugField(max_length=255) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) organization = models.ForeignKey( "api.Organization", @@ -76,14 +80,12 @@ class Publication(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - base_slug = slugify(self.title) or "untitled" - slug = base_slug - counter = 1 - while Publication.objects.filter(slug=slug).exclude(pk=self.pk).exists(): - slug = f"{base_slug}-{counter}" - counter += 1 - self.slug = slug - super().save(*args, **kwargs) + self.slug = slugify(self.title)[:255] or "untitled" + save_with_public_id(self, lambda: super(Publication, self).save(*args, **kwargs)) + + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_publication(self) -> bool: @@ -149,6 +151,7 @@ class PublicationBlock(models.Model): # FILE block fields. file = models.FileField( upload_to=_publication_block_directory_path, + storage=public_storage, max_length=300, blank=True, ) diff --git a/api/models/Resource.py b/api/models/Resource.py index 4a3a131f..f3f54d3a 100644 --- a/api/models/Resource.py +++ b/api/models/Resource.py @@ -1,23 +1,19 @@ import hashlib -import os -import random import uuid -from pathlib import Path from typing import TYPE_CHECKING, Any, Optional import structlog -from django.conf import settings from django.contrib.auth import get_user_model from django.db import models -from django.db.models.signals import post_save, pre_save +from django.db.models.signals import pre_save from django.dispatch import receiver from django.utils.text import slugify logger = structlog.getLogger(__name__) -from api.managers.dvc_manager import DVCManager +from api.utils.data_table_names import data_table_name from api.utils.enums import DataType -from api.utils.version_detection import detect_version_change_type +from api.utils.storage import private_storage if TYPE_CHECKING: from api.models.Dataset import Dataset @@ -46,7 +42,9 @@ class Resource(models.Model): ) name = models.CharField(max_length=200, unique=False, blank=False) description = models.TextField(blank=True, null=True) - slug = models.SlugField(max_length=255, unique=True) + # Not unique: nothing looks a resource up by slug, and a unique slug made + # simultaneous uploads of the same file name fail (#229). + slug = models.SlugField(max_length=255) url = models.URLField(max_length=500) is_active = models.BooleanField(default=True) preview_enabled = models.BooleanField(default=False) @@ -58,13 +56,7 @@ class Resource(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if not self.slug: - base_slug = slugify(self.name) - slug = base_slug - counter = 1 - while Resource.objects.filter(slug=slug).exclude(pk=self.pk).exists(): - slug = f"{base_slug}-{counter}" - counter += 1 - self.slug = slug + self.slug = slugify(self.name)[:255] super().save(*args, **kwargs) def __str__(self) -> str: @@ -73,7 +65,7 @@ def __str__(self) -> str: class ResourceFileDetails(models.Model): resource = models.OneToOneField(Resource, on_delete=models.CASCADE, null=False, blank=False) - file = models.FileField(upload_to="resources/", max_length=300) + file = models.FileField(upload_to="resources/", storage=private_storage, max_length=300) size = models.FloatField(blank=True, null=True) sha256 = models.CharField( max_length=64, @@ -110,11 +102,8 @@ def __str__(self): def save(self, *args, **kwargs): if not self.table_name: - # Generate a unique table name based on resource ID - safe_name = slugify(self.resource.name)[:50] - safe_title = slugify(self.resource.dataset.title)[:50] - resource_id = self.resource.id.hex[:8] # Use shorter ID - self.table_name = f"{safe_name}_{safe_title}_{resource_id}" + # Named from this row's own id: unique, and never cut by Postgres's 63-byte limit. + self.table_name = data_table_name(self.id) super().save(*args, **kwargs) @@ -168,156 +157,6 @@ def hash_resource_file(sender, instance: ResourceFileDetails, **kwargs): instance.sha256 = compute_sha256(instance.file) -@receiver(post_save, sender=ResourceFileDetails) -def version_resource_with_dvc(sender, instance: ResourceFileDetails, created, **kwargs): - """Create a new version using DVC when resource is updated""" - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - - # Skip if this is just being created (first version) - if created: - # For first version, we just track it - try: - # Use chunked mode for large files (over 100MB) - file_size = ( - instance.file.size - if hasattr(instance.file, "size") - else os.path.getsize(instance.file.path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - dvc_file = dvc.track_resource(instance.file.path, chunked=use_chunked) - message = f"Add resource: {instance.resource.name} version {instance.resource.version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{instance.resource.name}-{instance.resource.version}") - - # Create first version record - ResourceVersion.objects.create( - resource=instance.resource, - version_number=instance.resource.version, - change_description=f"Initial version of {instance.resource.name}", - ) - except Exception as e: - logger.error(f"Failed to version resource: {str(e)}") - # Continue without versioning if it fails - # This allows the resource to be created even if DVC fails - pass - else: - # For updates, check if the file has actually changed before creating a new version - try: - # Skip versioning if the file hasn't changed - if dvc.verify_file(instance.file.path): - logger.info( - f"No changes detected for {instance.resource.name}, skipping version creation" - ) - return - - # Get the latest version - last_version: Optional[ResourceVersion] = instance.resource.versions.order_by( - "-created_at" - ).first() - - # Handle case when there are no versions yet - if last_version is None: - new_version = "v1.0.0" - else: - # Determine appropriate version increment type by analyzing changes - try: - # Get the previous version file path - # We need to create a temporary copy of the previous version - import shutil - import tempfile - - from django.core.files.storage import default_storage - - # Create a temporary directory for the previous version - with tempfile.TemporaryDirectory() as temp_dir: - # Get the previous version file path - file_ext = instance.file.name.split(".")[-1] if instance.file.name else "" - prev_file_name = f"prev_version_{instance.resource.id}.{file_ext}" - prev_file_path = os.path.join(temp_dir, prev_file_name) - - # Use DVC to get the previous version - try: - # Try to checkout the previous version using DVC - rel_path = Path(instance.file.path).relative_to(settings.DVC_REPO_PATH) - tag_name = f"{instance.resource.name}-{last_version.version_number}" - - # Save current file to temp location - current_file_path = instance.file.path - temp_current = os.path.join(temp_dir, "current_file") - shutil.copy2(current_file_path, temp_current) - - # Checkout previous version - dvc.rollback_to_version(instance.file.path, tag_name) - - # Copy the previous version to our temp location - shutil.copy2(instance.file.path, prev_file_path) - - # Restore current file - shutil.copy2(temp_current, current_file_path) - - # Detect version change type - increment_type = detect_version_change_type( - prev_file_path, current_file_path - ) - logger.info( - f"Detected version change type: {increment_type} for {instance.resource.name}" - ) - except Exception as e: - logger.warning( - f"Could not analyze version changes: {str(e)}, defaulting to minor version" - ) - increment_type = "minor" - except Exception as e: - logger.warning( - f"Error in version detection: {str(e)}, defaulting to minor version" - ) - increment_type = "minor" - - # Increment version based on detected change type - new_version = _increment_version( - last_version.version_number, increment_type=increment_type - ) - - # Use chunked mode for large files (over 100MB) - file_size = ( - instance.file.size - if hasattr(instance.file, "size") - else os.path.getsize(instance.file.path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - # Update using DVC - dvc_file = dvc.track_resource(instance.file.path, chunked=use_chunked) - message = f"Update resource: {instance.resource.name} to version {new_version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{instance.resource.name}-{new_version}") - - # Create version record - ResourceVersion.objects.create( - resource=instance.resource, - version_number=new_version, - change_description=f"Updated version of {instance.resource.name}", - ) - - # Update resource version field - instance.resource.version = new_version - instance.resource.save(update_fields=["version"]) - - # Optional: Trigger garbage collection periodically - # This could be moved to a scheduled task instead - if random.random() < 0.05: # 5% chance to run GC on version update - try: - dvc.gc_cache() - except Exception as e: - logger.warning(f"Failed to run garbage collection: {str(e)}") - except Exception as e: - logger.error(f"Failed to update resource version: {str(e)}") - # Continue without versioning if it fails - pass - - def _increment_version(version: str, increment_type: str = "minor") -> str: """Semantic version incrementing logic diff --git a/api/models/ResourceChartImage.py b/api/models/ResourceChartImage.py index 16812dbb..9c19256d 100644 --- a/api/models/ResourceChartImage.py +++ b/api/models/ResourceChartImage.py @@ -6,6 +6,7 @@ # Import the Dataset model indirectly to avoid duplicate import error from api.utils.file_paths import _chart_image_directory_path +from api.utils.storage import private_storage class ResourceChartImage(models.Model): @@ -13,7 +14,11 @@ class ResourceChartImage(models.Model): name = models.CharField(max_length=50, unique=False, blank=True) description = models.CharField(max_length=1000, unique=False, blank=True) image = models.ImageField( - upload_to=_chart_image_directory_path, blank=True, null=True, max_length=300 + upload_to=_chart_image_directory_path, + storage=private_storage, + blank=True, + null=True, + max_length=300, ) dataset = models.ForeignKey( "api.Dataset", diff --git a/api/models/UseCase.py b/api/models/UseCase.py index 321bfdfc..517531a9 100644 --- a/api/models/UseCase.py +++ b/api/models/UseCase.py @@ -9,27 +9,34 @@ from api.models.Organization import Organization from authorization.models import User -from api.utils.django_utils import retry_on_slug_collision from api.utils.enums import ( OrganizationRelationshipType, UseCaseRunningStatus, UseCaseStatus, ) from api.utils.file_paths import _use_case_directory_path +from api.utils.public_links import PUBLIC_ID_LENGTH, build_url_slug, save_with_public_id +from api.utils.storage import public_storage class UseCase(models.Model): id = models.AutoField(primary_key=True) - title = models.CharField(max_length=200, unique=True, blank=True, null=True) + title = models.CharField(max_length=200, blank=True, null=True) summary = models.TextField(max_length=200_000, blank=True, null=True) logo = models.ImageField( - upload_to=_use_case_directory_path, max_length=300, blank=True, null=True + upload_to=_use_case_directory_path, + storage=public_storage, + max_length=300, + blank=True, + null=True, ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) website = models.URLField(blank=True) contact_email = models.EmailField(blank=True, null=True) - slug = models.SlugField(max_length=75, null=True, blank=True, unique=True) + # Kept for older readers; no longer unique. Links use public_id (see url_slug). + slug = models.SlugField(max_length=75, null=True, blank=True) + public_id = models.CharField(max_length=PUBLIC_ID_LENGTH, unique=True, editable=False) user = models.ForeignKey("authorization.User", on_delete=models.CASCADE) organization = models.ForeignKey( "api.Organization", on_delete=models.CASCADE, null=True, blank=True @@ -64,16 +71,12 @@ class UseCase(models.Model): def save(self, *args: Any, **kwargs: Any) -> None: if self.title and not self.slug: - self.slug = slugify(cast(str, self.title)) - base_title = self.title - - def disambiguate(attempt: int) -> None: - self.title = f"{base_title} ({attempt})" - self.slug = slugify(cast(str, self.title)) + self.slug = slugify(cast(str, self.title))[:75].strip("-") + save_with_public_id(self, lambda: super(UseCase, self).save(*args, **kwargs)) - retry_on_slug_collision( - lambda: super(UseCase, self).save(*args, **kwargs), disambiguate - ) + @property + def url_slug(self) -> str: + return build_url_slug(self.title, self.public_id) @property def is_individual_usecase(self): diff --git a/api/models/__init__.py b/api/models/__init__.py index 19f7c515..ebd4861a 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -11,6 +11,7 @@ from api.models.DatasetMetadata import DatasetMetadata from api.models.DatasetSource import DatasetSource from api.models.DataSpace import DataSpace +from api.models.Event import Event from api.models.Geography import Geography from api.models.Metadata import Metadata from api.models.Organization import Organization diff --git a/api/schema/access_model_schema.py b/api/schema/access_model_schema.py index 84ed8b09..6c320327 100644 --- a/api/schema/access_model_schema.py +++ b/api/schema/access_model_schema.py @@ -1,3 +1,5 @@ +# mypy: disable-error-code="valid-type" + import uuid from enum import Enum from typing import Any, Dict, List, Optional, Union @@ -16,6 +18,11 @@ ResourceSchema, ) from api.types.type_access_model import TypeAccessModel +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import AccessTypes AccessTypesEnum = strawberry.enum(AccessTypes) # type: ignore @@ -49,18 +56,38 @@ class EditAccessModelInput: @strawberry.type(name="Query") class Query: @strawberry_django.field - def access_model_resources( - self, info: Info, dataset_id: uuid.UUID - ) -> List[TypeAccessModel]: - models = AccessModel.objects.filter(dataset_id=dataset_id) + def access_model_resources(self, info: Info, dataset_id: uuid.UUID) -> List[TypeAccessModel]: + models = AccessModel.objects.filter(dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeAccessModel.from_django(model) for model in models] @strawberry_django.field def access_model(self, info: Info, access_model_id: uuid.UUID) -> TypeAccessModel: - model = AccessModel.objects.get(id=access_model_id) + model = AccessModel.objects.select_related("dataset").get(id=access_model_id) + if not can_view_dataset(info.context.user, model.dataset): + # Same answer as a missing ID. + raise AccessModel.DoesNotExist("AccessModel matching query does not exist.") return TypeAccessModel.from_django(model) +class ChangeAccessModelPermission(ChangeDatasetPartPermission): + """Access models belong to a dataset; changing one needs edit rights on it.""" + + message = "You don't have permission to change this dataset's access models" + + def datasets(self, kwargs: Any) -> Optional[list]: + data = kwargs.get("access_model_input") + access_model_id = kwargs.get("access_model_id") or getattr(data, "access_model_id", None) + if access_model_id: + # An existing access model stays in its own dataset whatever the input says. + model = AccessModel.objects.select_related("dataset").filter(id=access_model_id).first() + return [model.dataset] if model else None + dataset_id = getattr(data, "dataset", None) + dataset = Dataset.objects.filter(id=dataset_id).first() if dataset_id else None + return [dataset] if dataset else None + + def _add_resource_fields( access_model_resource: AccessModelResource, dataset_resource: Resource, @@ -86,18 +113,17 @@ def _add_update_access_model_resources( return for resource_input in model_input_resources: try: - dataset_resource = Resource.objects.get(id=resource_input.resource) - except Resource.DoesNotExist as e: - raise ValueError( - f"Resource with ID {resource_input.resource} does not exist." + # Only the access model's own dataset's resources. + dataset_resource = Resource.objects.get( + id=resource_input.resource, dataset_id=access_model.dataset_id ) + except Resource.DoesNotExist as e: + raise ValueError(f"Resource with ID {resource_input.resource} does not exist.") access_model_resource = AccessModelResource.objects.create( access_model=access_model, resource=dataset_resource ) - _add_resource_fields( - access_model_resource, dataset_resource, resource_input.fields - ) + _add_resource_fields(access_model_resource, dataset_resource, resource_input.fields) def _update_access_model_fields( @@ -115,29 +141,31 @@ def _update_access_model_fields( @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeAccessModelPermission] + ) def create_access_model( self, info: Info, access_model_input: AccessModelInput ) -> TypeAccessModel: try: dataset = Dataset.objects.get(id=access_model_input.dataset) except Dataset.DoesNotExist: - raise ValueError( - f"Dataset with ID {access_model_input.dataset} does not exist." - ) + raise ValueError(f"Dataset with ID {access_model_input.dataset} does not exist.") access_model = AccessModel.objects.create( dataset=dataset, name=access_model_input.name, description=access_model_input.description, - type=access_model_input.type.value, + type=access_model_input.type.value, # type: ignore[attr-defined] ) _update_access_model_fields(access_model, access_model_input) _add_update_access_model_resources(access_model, access_model_input.resources) return TypeAccessModel.from_django(access_model) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeAccessModelPermission] + ) def edit_access_model( self, info: Info, access_model_input: EditAccessModelInput ) -> TypeAccessModel: @@ -145,15 +173,11 @@ def edit_access_model( try: dataset = Dataset.objects.get(id=access_model_input.dataset) except Dataset.DoesNotExist as e: - raise ValueError( - f"Dataset with ID {access_model_input.dataset} does not exist." - ) + raise ValueError(f"Dataset with ID {access_model_input.dataset} does not exist.") access_model = AccessModel.objects.create(dataset=dataset) else: try: - access_model = AccessModel.objects.get( - id=access_model_input.access_model_id - ) + access_model = AccessModel.objects.get(id=access_model_input.access_model_id) except AccessModel.DoesNotExist as e: raise ValueError( f"Access Model with ID {access_model_input.access_model_id} does not exist." @@ -163,7 +187,9 @@ def edit_access_model( _add_update_access_model_resources(access_model, access_model_input.resources) return TypeAccessModel.from_django(access_model) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeAccessModelPermission] + ) def delete_access_model(self, info: Info, access_model_id: uuid.UUID) -> bool: try: access_model = AccessModel.objects.get(id=access_model_id) diff --git a/api/schema/aimodel_schema.py b/api/schema/aimodel_schema.py index f2075a22..7c4655be 100644 --- a/api/schema/aimodel_schema.py +++ b/api/schema/aimodel_schema.py @@ -1,13 +1,14 @@ """GraphQL schema for AI Model.""" -# mypy: disable-error-code="union-attr,misc" +# mypy: disable-error-code="union-attr,misc,valid-type" import datetime -from typing import List, Optional +from typing import Any, Dict, List, Optional import strawberry import strawberry_django from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet from strawberry.types import Info from strawberry_django.pagination import OffsetPaginationInput @@ -21,6 +22,7 @@ AIModelLifecycleStageEnum, AIModelOrder, AIModelProviderEnum, + AIModelResponse, AIModelStatusEnum, AIModelTypeEnum, AIModelVersionFilter, @@ -34,6 +36,13 @@ TypeVersionProvider, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.graphql_permissions import IsAuthenticated from authorization.models import OrganizationMembership, Role @@ -281,6 +290,57 @@ class UpdateVersionProviderInput: config: Optional[strawberry.scalars.JSON] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +AI_MODELS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "display_name", + "status", + "model_type", + "provider", + "domain", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "created_at", + "updated_at", +] +AI_MODELS_TABLE_SORT_FIELDS = [ + "name", + "display_name", + "created_at", + "updated_at", + "status", + "model_type", + "provider", +] + + +def _visible_ai_models(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The AI models a caller may list. Same rules as `aiModels`.""" + organization = info.context.context.get("organization") + user = info.context.user + public = AIModel.objects.filter(is_public=True, is_active=True) + if organization: + queryset = AIModel.objects.filter(organization=organization) + elif not user.is_authenticated: + queryset = public + elif user.is_superuser: + queryset = AIModel.objects.all() + else: + queryset = AIModel.objects.filter(user=user, organization=None) + if include_public: + queryset = queryset | public + return queryset.distinct() + + @strawberry.type class Query: """Queries for AI Models.""" @@ -330,13 +390,60 @@ def ai_models( return TypeAIModel.from_django_list(list(queryset)) + @strawberry.field + @trace_resolver(name="ai_models_table", attributes={"component": "aimodel"}) + def ai_models_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> AIModelResponse: + """AI models for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `aiModels`, with the same visibility rules, plus + public, active models when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_ai_models(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=AI_MODELS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created_at", direction="desc")], + sorting_allowed_fields=AI_MODELS_TABLE_SORT_FIELDS, + ) + return AIModelResponse( + data=TypeAIModel.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, AI_MODELS_TABLE_FILTER_FIELDS), + ) + @strawberry.field @trace_resolver(name="get_ai_model", attributes={"component": "aimodel"}) - def get_ai_model(self, info: Info, model_id: int) -> Optional[TypeAIModel]: - """Get an AI model by ID.""" + def get_ai_model( + self, + info: Info, + model_id: Optional[int] = None, + public_id: Optional[str] = None, + ) -> Optional[TypeAIModel]: + """Get an AI model by ID, or by its link (``urlSlug``) or ``publicId``.""" user = info.context.user try: - model = AIModel.objects.get(id=model_id) + if model_id is not None: + model = AIModel.objects.get(id=model_id) + else: + parsed = parse_public_id(public_id) + if parsed is None: + return None + model = AIModel.objects.get(public_id=parsed) # Check permissions if model.is_public and model.is_active: @@ -419,14 +526,14 @@ def create_ai_model( description = input.description or "" # Prepare supported_languages - supported_languages = input.supported_languages or [] + supported_languages: List[str] = input.supported_languages or [] # Prepare schemas - input_schema = input.input_schema or {} - output_schema = input.output_schema or {} + input_schema: Any = input.input_schema or {} + output_schema: Any = input.output_schema or {} # Prepare metadata - metadata = input.metadata or {} + metadata: Any = input.metadata or {} try: model = AIModel.objects.create( @@ -802,7 +909,7 @@ def create_ai_model_version( ai_model=model, version=input.version, version_notes=input.version_notes or "", - lifecycle_stage=input.lifecycle_stage.value if input.lifecycle_stage else "DEVELOPMENT", # type: ignore[misc] + lifecycle_stage=input.lifecycle_stage.value if input.lifecycle_stage else "DEVELOPMENT", # type: ignore[attr-defined] supports_streaming=input.supports_streaming, max_tokens=input.max_tokens, supported_languages=input.supported_languages or [], diff --git a/api/schema/collaborative_schema.py b/api/schema/collaborative_schema.py index 366f4aa6..a1eca2d2 100644 --- a/api/schema/collaborative_schema.py +++ b/api/schema/collaborative_schema.py @@ -9,6 +9,7 @@ import strawberry import strawberry_django from django.db import models +from django.db.models import QuerySet from strawberry import auto from strawberry.file_uploads import Upload from strawberry.types import Info @@ -37,6 +38,7 @@ from api.types.type_collaborative import ( CollaborativeFilter, CollaborativeOrder, + CollaborativeResponse, TypeCollaborative, ) from api.types.type_collaborative_organization import ( @@ -48,6 +50,13 @@ from api.types.type_usecase import TypeUseCase from api.utils.enums import CollaborativeStatus, OrganizationRelationshipType from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.models import User from authorization.types import TypeUser @@ -95,6 +104,55 @@ class CollaborativeInputPartial: completed_on: Optional[datetime.date] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +COLLABORATIVES_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "sdgs__code", + "sdgs__slug", + "started_on", + "completed_on", + "created", + "modified", +] +COLLABORATIVES_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "started_on", + "completed_on", +] + + +def _visible_collaboratives(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The collaboratives a caller may list. Same rules as `collaboratives`.""" + user = info.context.user + organization = info.context.context.get("organization") + if organization: + queryset = Collaborative.objects.filter(organization=organization) + elif user.is_superuser: + queryset = Collaborative.objects.all() + elif user.is_authenticated: + queryset = Collaborative.objects.filter(user=user) + else: + queryset = Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) + if include_public: + queryset = queryset | Collaborative.objects.filter(status=CollaborativeStatus.PUBLISHED) + return queryset.distinct() + + @strawberry.type(name="Query") class Query: """Queries for collaboratives.""" @@ -138,6 +196,42 @@ def collaboratives( return TypeCollaborative.from_django_list(queryset) + @strawberry.field + @trace_resolver(name="collaboratives_table", attributes={"component": "collaborative"}) + def collaboratives_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> CollaborativeResponse: + """Collaboratives for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `collaboratives`, with the same visibility rules, plus + published collaboratives when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_collaboratives(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=COLLABORATIVES_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=COLLABORATIVES_TABLE_SORT_FIELDS, + ) + return CollaborativeResponse( + data=TypeCollaborative.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, COLLABORATIVES_TABLE_FILTER_FIELDS), + ) + @strawberry_django.field @trace_resolver(name="get_published_collaboratives", attributes={"component": "collaborative"}) def published_collaboratives( @@ -199,12 +293,24 @@ def contributors_by_collaborative(self, info: Info, collaborative_id: str) -> li name="get_collaborative_by_slug", attributes={"component": "collaborative"}, ) - def collaborative_by_slug(self, info: Info, slug: str) -> TypeCollaborative: - try: - collaborative = Collaborative.objects.get(slug=slug) - return TypeCollaborative.from_django(collaborative) - except Collaborative.DoesNotExist: - raise ValueError(f"Collaborative with slug {slug} does not exist.") + def collaborative_by_slug( + self, info: Info, public_id: Optional[str] = None, slug: Optional[str] = None + ) -> TypeCollaborative: + """Collaborative for its page, found by its link. + + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. ``slug`` is for links and subdomains made before public ids + existed: the id is tried first, then the old slug. + """ + value = public_id if public_id is not None else slug + parsed = parse_public_id(value) + collaborative = Collaborative.objects.filter(public_id=parsed).first() if parsed else None + if collaborative is None and public_id is None and slug: + collaborative = Collaborative.objects.filter(slug=slug).order_by("created").first() + if collaborative is None: + raise ValueError(f"Collaborative {value} does not exist.") + return TypeCollaborative.from_django(collaborative) @trace_resolver(name="update_collaborative_tags", attributes={"component": "collaborative"}) diff --git a/api/schema/dataset_schema.py b/api/schema/dataset_schema.py index edce62c8..85317caf 100644 --- a/api/schema/dataset_schema.py +++ b/api/schema/dataset_schema.py @@ -1,4 +1,5 @@ # mypy: disable-error-code=union-attr +# mypy: disable-error-code=valid-type import datetime import uuid from typing import Any, List, Optional, Union @@ -43,6 +44,11 @@ from api.types.type_resource import TypeResource from api.types.type_resource_chart import TypeResourceChart from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_change_dataset, + can_list_organization, +) from api.utils.enums import ( DatasetAccessType, DatasetLicense, @@ -54,11 +60,12 @@ UseCaseStatus, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id from api.utils.qs_utils import ( FilterSpec, SortSpec, - apply_filters, get_pagination_window, + status_counts, ) from authorization.models import DatasetPermission, OrganizationMembership, Role, User from authorization.permissions import ( @@ -76,6 +83,14 @@ PromptPurposeENUM = strawberry.enum(PromptPurpose) # type: ignore +def _find_dataset(dataset_id: Optional[uuid.UUID], public_id: Optional[str]) -> Optional[Dataset]: + """The dataset for a UUID or a public link part, or None if there is none.""" + if dataset_id: + return Dataset.objects.filter(id=dataset_id).first() + parsed = parse_public_id(public_id) + return Dataset.objects.filter(public_id=parsed).first() if parsed else None + + # Create permission classes dynamically with different operations class ViewDatasetPermission(DatasetPermissionGraphQL): def __init__(self) -> None: @@ -113,19 +128,19 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: # For queries/mutations that don't have a source yet (e.g., getting a dataset by ID) if source is None: dataset_id = kwargs.get("dataset_id") - if dataset_id: - try: - dataset = Dataset.objects.get(id=dataset_id) - # Allow access to published datasets for everyone - if dataset.status == DatasetStatus.PUBLISHED.value: - return True - except Dataset.DoesNotExist: - pass # Let the resolver handle the non-existent dataset + public_id = kwargs.get("public_id") + if dataset_id or public_id: + dataset = _find_dataset(dataset_id, public_id) + # Allow access to published datasets for everyone + if dataset is not None and dataset.status == DatasetStatus.PUBLISHED.value: + return True # For non-published datasets, require authentication user = request.user if not user.is_authenticated: return False + if dataset is None: + return True # the resolver returns null for a missing dataset if user.is_superuser: return True dataset_perm = DatasetPermission.objects.filter(user=user, dataset=dataset).first() @@ -227,38 +242,25 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: if not update_dataset_input or not hasattr(update_dataset_input, "dataset"): return False - dataset_id = update_dataset_input.dataset - - try: - dataset = Dataset.objects.get(id=dataset_id) - - # Check if user owns the dataset - if dataset.user and dataset.user == user: - return True - - # If organization-owned, check organization permissions - if dataset.organization: - # Get the roles with names 'admin' or 'editor' - admin_editor_roles = Role.objects.filter( - name__in=["admin", "editor", "owner"] - ).values_list("id", flat=True) + dataset = Dataset.objects.filter(id=update_dataset_input.dataset).first() + if not dataset: + return False + return can_change_dataset(user, dataset) - # Check if user is a member of the dataset's organization with appropriate role - org_member = OrganizationMembership.objects.filter( - user=user, - organization=dataset.organization, - role__id__in=admin_editor_roles, - ).exists() - if org_member: - return True +class ChangePromptResourcePermission(ChangeDatasetPartPermission): + """Prompt details of a file need edit rights on the file's dataset.""" - # Check dataset-specific permissions - dataset_perm = DatasetPermission.objects.filter(user=user, dataset=dataset).first() - return dataset_perm and dataset_perm.role.can_change and dataset.status == DatasetStatus.DRAFT.value # type: ignore + message = "You don't have permission to update this dataset" - except Dataset.DoesNotExist: - return False + def datasets(self, kwargs: Any) -> Optional[list]: + resource_id = getattr(kwargs.get("update_input"), "resource", None) + resource = ( + Resource.objects.select_related("dataset").filter(id=resource_id).first() + if resource_id + else None + ) + return [resource.dataset] if resource else None @strawberry.input @@ -434,7 +436,11 @@ def _visible_datasets(info: Info, include_public: Optional[bool] = False) -> Que organization = info.context.context.get("organization") user = info.context.user if organization: - queryset = Dataset.objects.filter(organization=organization) + queryset = ( + Dataset.objects.filter(organization=organization) + if can_list_organization(user, organization) + else Dataset.objects.none() + ) elif not user.is_authenticated: queryset = Dataset.objects.none() elif user.is_superuser: @@ -463,24 +469,17 @@ def datasets( user = info.context.user if organization: - queryset = Dataset.objects.filter(organization=organization) + # Only members whose role can view may list an organisation's datasets + if can_list_organization(user, organization): + queryset = Dataset.objects.filter(organization=organization) + else: + queryset = Dataset.objects.none() else: # If user is authenticated if user.is_authenticated: # If user is superuser, show all datasets if user.is_superuser: queryset = Dataset.objects.all() - elif organization: - # Check id user has access to organization - org_member = OrganizationMembership.objects.get( - user=user, organization=organization - ) - if org_member.exists() and org_member.role.can_view: # type: ignore - # Show only datasets from current organization - queryset = Dataset.objects.filter(organization=organization) - else: - # if user is not a member of the organization, return empty queryset - queryset = Dataset.objects.none() else: # For non-organization authenticated users, only owned datasets queryset = Dataset.objects.filter(user=user, organization=None) @@ -538,33 +537,31 @@ def datasets_table( sort_options=sort_options or [SortSpec(field="created", direction="desc")], sorting_allowed_fields=DATASETS_TABLE_SORT_FIELDS, ) - # Tab labels: counts per status under every filter except status itself. - without_status = [f for f in (filters or []) if f.field.strip() != "status"] - status_rows = ( - apply_filters(visible, without_status, DATASETS_TABLE_FILTER_FIELDS) - .order_by() - .values("status") - .annotate(n=Count("id", distinct=True)) - ) return DatasetResponse( data=TypeDataset.from_django_list(window["data"]), total_items_count=window["total_items_count"], - status_counts=[ - DatasetStatusCount(status=row["status"], count=row["n"]) for row in status_rows - ], + status_counts=status_counts( + visible, filters, DATASETS_TABLE_FILTER_FIELDS, item_type=DatasetStatusCount + ), ) @strawberry.field( permission_classes=[AllowPublishedDatasets], # type: ignore[list-item] ) @trace_resolver(name="get_dataset", attributes={"component": "dataset"}) - def get_dataset(self, info: Info, dataset_id: uuid.UUID) -> Optional[TypeDataset]: - """Get a dataset by ID.""" - try: - dataset = Dataset.objects.get(id=dataset_id) - return TypeDataset.from_django(dataset) - except Dataset.DoesNotExist: - return None + def get_dataset( + self, + info: Info, + dataset_id: Optional[uuid.UUID] = None, + public_id: Optional[str] = None, + ) -> Optional[TypeDataset]: + """Get a dataset by ID, or by its public link. + + ``public_id`` takes the id itself or the whole link part + (``city-air-quality-index-k3x9m2qa7b1z``); only the id at the end is used. + """ + dataset = _find_dataset(dataset_id, public_id) + return TypeDataset.from_django(dataset) if dataset is not None else None @strawberry.field( permission_classes=[ChartDataPermission], # type: ignore[list-item] @@ -862,7 +859,7 @@ def update_prompt_metadata( return MutationResponse.success_response(TypePromptDataset.from_django(prompt_dataset)) @strawberry.mutation( - permission_classes=[IsAuthenticated], + permission_classes=[ChangePromptResourcePermission], ) @trace_resolver( name="update_prompt_resource", diff --git a/api/schema/event_schema.py b/api/schema/event_schema.py new file mode 100644 index 00000000..95a85034 --- /dev/null +++ b/api/schema/event_schema.py @@ -0,0 +1,302 @@ +"""Schema definitions for events. + +Resolvers delegate metadata checks, writes, and scoping to +``api/services/event_service.py``. +""" + +import datetime +import uuid +from typing import List, Optional + +import strawberry +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet +from strawberry.types import Info + +from api.models import Event +from api.schema.base_mutation import BaseMutation, MutationResponse +from api.services.event_service import ( + apply_event_update, + assert_ready_to_publish, + create_event, + get_scoped_events, + set_event_status, +) +from api.types.type_event import EventResponse, TypeEvent +from api.utils.enums import EventStatus +from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) +from authorization.permissions import ( + AllowPublishedEvents, + ChangeEventPermission, + CreateEventPermission, + DeleteEventPermission, + PublishEventPermission, +) + + +@strawberry.input +class CreateEventInput: + """Fields for a new draft. Every field is optional.""" + + name: Optional[str] = None + location: Optional[str] = None + start_date: Optional[datetime.date] = None + end_date: Optional[datetime.date] = None + luma_embed: Optional[str] = None + + +@strawberry.input +class UpdateEventInput: + """Partial edit. ``UNSET`` means the client omitted the field.""" + + id: uuid.UUID + name: Optional[str] = strawberry.UNSET + location: Optional[str] = strawberry.UNSET + start_date: Optional[datetime.date] = strawberry.UNSET + end_date: Optional[datetime.date] = strawberry.UNSET + luma_embed: Optional[str] = strawberry.UNSET + + +EVENTS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "location", + "status", + "start_date", + "end_date", + "organization", + "organization__slug", + "created", + "modified", +] +EVENTS_TABLE_SORT_FIELDS = [ + "name", + "location", + "status", + "start_date", + "end_date", + "created", + "modified", +] + + +def _visible_events(info: Info) -> QuerySet: + return get_scoped_events( + user=info.context.user, + organization=info.context.context.get("organization"), + ) + + +@strawberry.type(name="Query") +class Query: + @strawberry.field(permission_classes=[AllowPublishedEvents]) # type: ignore[list-item] + @trace_resolver(name="get_event", attributes={"component": "event"}) + def event(self, info: Info, event_id: uuid.UUID) -> Optional[TypeEvent]: + """Get one event. Drafts are gated to the owner or organization.""" + try: + return TypeEvent.from_django(Event.objects.get(id=event_id)) + except Event.DoesNotExist: + return None + + @strawberry.field + @trace_resolver(name="events_table", attributes={"component": "event"}) + def events_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + ) -> EventResponse: + """Events for the dashboard table, scoped like the other providers.""" + visible = _visible_events(info) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=EVENTS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="modified", direction="desc")], + sorting_allowed_fields=EVENTS_TABLE_SORT_FIELDS, + ) + return EventResponse( + data=TypeEvent.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, EVENTS_TABLE_FILTER_FIELDS), + ) + + @strawberry.field + @trace_resolver(name="published_events", attributes={"component": "event"}) + def published_events( + self, + info: Info, + organization_slug: Optional[str] = None, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 20, + offset: int = 0, + ) -> EventResponse: + """Published events for public pages. Drafts are never included.""" + visible = Event.objects.filter(status=EventStatus.PUBLISHED).select_related( + "organization", "user" + ) + if organization_slug: + visible = visible.filter(organization__slug=organization_slug) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=EVENTS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="start_date", direction="asc")], + sorting_allowed_fields=EVENTS_TABLE_SORT_FIELDS, + ) + return EventResponse( + data=TypeEvent.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=[], + ) + + @strawberry.field + @trace_resolver(name="event_by_slug", attributes={"component": "event"}) + def event_by_slug(self, info: Info, public_id: str) -> Optional[TypeEvent]: + """Published event for a public page, found by its link. + + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. Drafts, unknown ids and links without an id return null. + """ + parsed = parse_public_id(public_id) + if parsed is None: + return None + event = ( + Event.objects.filter(public_id=parsed, status=EventStatus.PUBLISHED) + .select_related("organization", "user") + .first() + ) + if event is None: + return None + return TypeEvent.from_django(event) + + +@strawberry.type(name="Mutation") +class Mutation: + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[CreateEventPermission], + trace_name="create_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "created", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def create_event(self, info: Info, input: CreateEventInput) -> MutationResponse[TypeEvent]: + """Create a draft owned by the caller's organization or the user.""" + event = create_event( + user=info.context.user, + organization=info.context.context.get("organization"), + name=input.name or "", + location=input.location or "", + start_date=input.start_date, + end_date=input.end_date, + luma_embed=input.luma_embed, + ) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[ChangeEventPermission], + trace_name="update_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "updated", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def update_event(self, info: Info, input: UpdateEventInput) -> MutationResponse[TypeEvent]: + """Apply a partial edit. Completeness is required only after publish.""" + event = _get_event_or_raise(input.id) + event = apply_event_update(event, **_update_kwargs(input)) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishEventPermission], + trace_name="publish_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "published", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def publish_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[TypeEvent]: + """Publish an event once name, location, dates, and a Luma embed are set.""" + event = _get_event_or_raise(event_id) + assert_ready_to_publish(event) + event = set_event_status(event, EventStatus.PUBLISHED) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[PublishEventPermission], + trace_name="unpublish_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "unpublished", + "get_data": lambda result, **kwargs: {"event_id": str(result.id)}, + }, + ) + def unpublish_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[TypeEvent]: + """Move an event back to draft.""" + event = _get_event_or_raise(event_id) + event = set_event_status(event, EventStatus.DRAFT) + return MutationResponse.success_response(TypeEvent.from_django(event)) + + @strawberry.mutation + @BaseMutation.mutation( + permission_classes=[DeleteEventPermission], + trace_name="delete_event", + trace_attributes={"component": "event"}, + track_activity={ + "verb": "deleted", + "get_data": lambda result, **kwargs: { + "event_id": str(kwargs.get("event_id")), + "success": result, + }, + }, + ) + def delete_event(self, info: Info, event_id: uuid.UUID) -> MutationResponse[bool]: + """Hard-delete an event.""" + event = _get_event_or_raise(event_id) + event.delete() + return MutationResponse.success_response(True) + + +def _update_kwargs(input: UpdateEventInput) -> dict: + kwargs: dict = {} + if input.name is not strawberry.UNSET: + kwargs["name"] = input.name + if input.location is not strawberry.UNSET: + kwargs["location"] = input.location + if input.start_date is not strawberry.UNSET: + kwargs["start_date"] = input.start_date + if input.end_date is not strawberry.UNSET: + kwargs["end_date"] = input.end_date + if input.luma_embed is not strawberry.UNSET: + kwargs["luma_embed"] = input.luma_embed + return kwargs + + +def _get_event_or_raise(event_id: uuid.UUID) -> Event: + try: + return Event.objects.get(id=event_id) + except Event.DoesNotExist: + raise DjangoValidationError(f"Event with id {event_id} does not exist.") diff --git a/api/schema/publication_schema.py b/api/schema/publication_schema.py index 8b0e354a..4631ba80 100644 --- a/api/schema/publication_schema.py +++ b/api/schema/publication_schema.py @@ -16,6 +16,7 @@ import strawberry import strawberry_django from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import QuerySet from strawberry.file_uploads import Upload from strawberry.types import Info @@ -41,6 +42,7 @@ from api.types.type_publication import ( PublicationFilter, PublicationOrder, + PublicationResponse, TypePublication, TypePublicationBlock, TypeResourceType, @@ -48,12 +50,19 @@ ) from api.utils.enums import PublicationStatus from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.permissions import ( AllowPublishedPublications, ChangePublicationPermission, CreatePublicationPermission, DeletePublicationPermission, PublishPublicationPermission, + find_publication, ) @@ -93,6 +102,46 @@ class UpdatePublicationInput: external_source_link: Optional[str] = strawberry.UNSET +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +PUBLICATIONS_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "license", + "resource_type", + "resource_type__slug", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "publication_date", + "created", + "modified", +] +PUBLICATIONS_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "publication_date", + "download_count", +] + + +def _visible_publications(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The publications a caller may list. Same rules as `publications`.""" + return get_scoped_publications( + user=info.context.user, + organization=info.context.context.get("organization"), + include_public=bool(include_public), + ) + + @strawberry.type(name="Query") class Query: """Queries for publications.""" @@ -109,12 +158,18 @@ def resource_types(self, info: Info) -> List[TypeResourceType]: permission_classes=[AllowPublishedPublications], # type: ignore[list-item] ) @trace_resolver(name="get_publication", attributes={"component": "publication"}) - def get_publication(self, info: Info, publication_id: uuid.UUID) -> Optional[TypePublication]: - """Get a single resource by id (drafts gated to owner/org by the permission).""" - try: - return TypePublication.from_django(Publication.objects.get(id=publication_id)) - except Publication.DoesNotExist: - return None + def get_publication( + self, + info: Info, + publication_id: Optional[uuid.UUID] = None, + public_id: Optional[str] = None, + ) -> Optional[TypePublication]: + """Get a single resource by id, or by its link (``urlSlug``) or ``publicId``. + + Drafts are gated to owner/org by the permission. + """ + publication = find_publication(publication_id, public_id) + return TypePublication.from_django(publication) if publication else None @strawberry.field @trace_resolver(name="get_publications", attributes={"component": "publication"}) @@ -147,6 +202,42 @@ def publications( ) return TypePublication.from_django_list(queryset[offset : offset + limit]) + @strawberry.field + @trace_resolver(name="publications_table", attributes={"component": "publication"}) + def publications_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> PublicationResponse: + """Publications for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `publications`, with the same visibility rules, plus + published publications when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_publications(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=PUBLICATIONS_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=PUBLICATIONS_TABLE_SORT_FIELDS, + ) + return PublicationResponse( + data=TypePublication.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, PUBLICATIONS_TABLE_FILTER_FIELDS), + ) + @strawberry.type(name="Mutation") class Mutation: diff --git a/api/schema/resource_chart_schema.py b/api/schema/resource_chart_schema.py index acb34b78..5eed2873 100644 --- a/api/schema/resource_chart_schema.py +++ b/api/schema/resource_chart_schema.py @@ -1,37 +1,215 @@ +# mypy: disable-error-code=valid-type + import datetime import uuid from typing import Any, Dict, List, Optional import strawberry import strawberry_django +from django.db.models import CharField, QuerySet, Value from strawberry.types import Info from strawberry_django.mutations import mutations -from api.models import Resource, ResourceChartDetails, ResourceSchema +from api.models import ( + Resource, + ResourceChartDetails, + ResourceChartImage, + ResourceSchema, +) +from api.schema.dataset_schema import ChartDataPermission from api.schema.extensions import TrackModelActivity -from api.types.type_resource_chart import TypeResourceChart +from api.types.type_resource_chart import ChartResponse, TypeResourceChart +from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import AggregateType, ChartStatus, ChartTypes +from api.utils.graphql_telemetry import trace_resolver +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + StatusCount, + apply_filters, + get_pagination_window, + status_counts, +) ChartTypeEnum = strawberry.enum(ChartTypes) AggregateTypeEnum = strawberry.enum(AggregateType) +# Fields chartsTable accepts. Names are the chart image's; a built chart reaches +# its dataset through its resource, so `dataset…` is rewritten for it. +# `kind` is CHART (built in the wizard) or IMAGE (uploaded). +CHARTS_TABLE_FILTER_FIELDS = [ + "id", + "name", + "status", + "chart_type", + "kind", + "dataset", + "dataset__slug", + "dataset__title", + "modified", +] +CHARTS_TABLE_SORT_FIELDS = ["name", "status", "chart_type", "kind", "modified"] + +# Columns shared by both tables for the UNION. Model fields first, then the +# annotations, so both sides select them in the same order. +_CHART_COLUMNS = ("id", "name", "status", "modified", "chart_type", "kind") + + +def _for_built_charts(field: str) -> str: + if field == "dataset" or field.startswith("dataset__"): + return f"resource__{field}" + return field + + +def _visible_charts(info: Info) -> "tuple[QuerySet, QuerySet]": + """Built charts and chart images the caller may list: the same rules as + `getChartData` without a dataset (the organisation's, else the user's).""" + organization = info.context.context.get("organization") + user = info.context.user + if organization: + charts = ResourceChartDetails.objects.filter(resource__dataset__organization=organization) + images = ResourceChartImage.objects.filter(dataset__organization=organization) + else: + charts = ResourceChartDetails.objects.filter(resource__dataset__user=user) + images = ResourceChartImage.objects.filter(dataset__user=user) + charts = charts.annotate(kind=Value("CHART", output_field=CharField())) + images = images.annotate( + chart_type=Value(None, output_field=CharField()), + kind=Value("IMAGE", output_field=CharField()), + ) + return charts, images + @strawberry.type(name="Query") class Query: @strawberry_django.field - def charts_details( - self, info: Info, dataset_id: uuid.UUID - ) -> List[TypeResourceChart]: - charts = ResourceChartDetails.objects.filter(resource__dataset_id=dataset_id) + def charts_details(self, info: Info, dataset_id: uuid.UUID) -> List[TypeResourceChart]: + charts = ResourceChartDetails.objects.filter(resource__dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="resource__dataset__") + ) return [TypeResourceChart.from_django(chart) for chart in charts] @strawberry_django.field - def resource_chart( - self, info: Info, chart_details_id: uuid.UUID - ) -> TypeResourceChart: - chart = ResourceChartDetails.objects.get(id=chart_details_id) + def resource_chart(self, info: Info, chart_details_id: uuid.UUID) -> TypeResourceChart: + chart = ResourceChartDetails.objects.select_related("resource__dataset").get( + id=chart_details_id + ) + if not can_view_dataset(info.context.user, chart.resource.dataset): + # Same answer as a missing ID. + raise ResourceChartDetails.DoesNotExist( + "ResourceChartDetails matching query does not exist." + ) return TypeResourceChart.from_django(chart) + @strawberry.field( + permission_classes=[ChartDataPermission], # type: ignore[list-item] + ) + @trace_resolver(name="charts_table", attributes={"component": "chart"}) + def charts_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + ) -> ChartResponse: + """Built charts and chart images in one table: client-driven filters, + sortable columns and page controls backed by a real total count. + + Additive next to `getChartData`, with the same visibility rules. Both + tables are merged in one SQL UNION, so counting, sorting and slicing + happen in the database; only the rows of the page are then loaded. + Default order: last modified first. + """ + visible_charts, visible_images = _visible_charts(info) + # Images first: their field names are the public ones, so a disallowed + # field is reported with the allowlist as clients know it. + images = apply_filters(visible_images, filters or [], CHARTS_TABLE_FILTER_FIELDS) + chart_filters = [ + FilterSpec(field=_for_built_charts(f.field or ""), condition=f.condition, value=f.value) + for f in filters or [] + ] + chart_fields = [_for_built_charts(f) for f in CHARTS_TABLE_FILTER_FIELDS] + charts = apply_filters(visible_charts, chart_filters, chart_fields) + + # order_by() drops the models' default ordering, not allowed inside a UNION. + rows = ( + charts.order_by() + .values(*_CHART_COLUMNS) + .union(images.order_by().values(*_CHART_COLUMNS)) + ) + # Filters are already applied per table above (not allowed after a UNION). + window = get_pagination_window( + rows, + limit=limit, + offset=offset, + sort_options=sort_options or [SortSpec(field="modified", direction="desc")], + sorting_allowed_fields=CHARTS_TABLE_SORT_FIELDS, + ) + page = list(window["data"]) + + # from_django reads every foreign key of the chart's dataset. + built = ResourceChartDetails.objects.select_related( + "resource__dataset__organization", + "resource__dataset__user", + "resource__dataset__dataspace", + ).in_bulk([r["id"] for r in page if r["kind"] == "CHART"]) + uploaded = ResourceChartImage.objects.select_related("dataset").in_bulk( + [r["id"] for r in page if r["kind"] == "IMAGE"] + ) + data: List[Any] = [] + for r in page: + if r["kind"] == "CHART" and r["id"] in built: + data.append(TypeResourceChart.from_django(built[r["id"]])) + elif r["kind"] == "IMAGE" and r["id"] in uploaded: + data.append(TypeResourceChartImage.from_django(uploaded[r["id"]])) + + counts: Dict[str, int] = {} + for item in status_counts(visible_charts, chart_filters, chart_fields) + status_counts( + visible_images, filters, CHARTS_TABLE_FILTER_FIELDS + ): + counts[item.status] = counts.get(item.status, 0) + item.count + return ChartResponse( + data=data, + total_items_count=window["total_items_count"], + status_counts=[StatusCount(status=k, count=v) for k, v in sorted(counts.items())], + ) + + +class ChangeChartPermission(ChangeDatasetPartPermission): + """Charts belong to a resource's dataset; changing one needs edit rights on it. + + Editing a chart onto another resource needs rights on both datasets. + """ + + message = "You don't have permission to change this dataset's charts" + + def datasets(self, kwargs: Any) -> Optional[list]: + chart_input = kwargs.get("chart_input") + found = [] + for resource_id in (kwargs.get("resource"), getattr(chart_input, "resource", None)): + if resource_id: + resource = Resource.objects.select_related("dataset").filter(id=resource_id).first() + if not resource: + return None + found.append(resource.dataset) + for chart_id in (kwargs.get("chart_id"), getattr(chart_input, "chart_id", None)): + if chart_id: + chart = ( + ResourceChartDetails.objects.select_related("resource__dataset") + .filter(id=chart_id) + .first() + ) + if not chart: + return None + found.append(chart.resource.dataset) + return found + @strawberry.input class FilterInput: @@ -182,20 +360,18 @@ def _update_chart_fields( "geospatial_field", ]: if value: # Only process if value is not empty - field = ResourceSchema.objects.get(id=value) + field = ResourceSchema.objects.get(id=value, resource=resource) options[field_name] = field elif field_name == "y_axis_column": if value: # Only process if list is not empty options[field_name] = [ { "field": ResourceSchema.objects.get( - id=column.field_name + id=column.field_name, resource=resource ), "label": column.label, "color": column.color, - "value_mapping": _update_value_mapping( - column.value_mapping - ), + "value_mapping": _update_value_mapping(column.value_mapping), } for column in value ] @@ -214,7 +390,7 @@ def _update_chart_fields( for filter_input in chart_input.filters: if filter_input.column: filter_dict = { - "column": ResourceSchema.objects.get(id=filter_input.column), + "column": ResourceSchema.objects.get(id=filter_input.column, resource=resource), "operator": filter_input.operator, "value": filter_input.value, } @@ -225,7 +401,9 @@ def _update_chart_fields( @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartPermission] + ) def add_resource_chart(self, info: Info, resource: uuid.UUID) -> TypeResourceChart: try: resource_obj = Resource.objects.get(id=resource) @@ -241,6 +419,7 @@ def add_resource_chart(self, info: Info, resource: uuid.UUID) -> TypeResourceCha @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeChartPermission], extensions=[ # Use TrackModelActivity to automatically track the activity # The extension will use the returned chart as the action object @@ -280,6 +459,7 @@ def create_resource_chart( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeChartPermission], extensions=[ # Track updates to charts TrackModelActivity( @@ -295,9 +475,7 @@ def create_resource_chart( ) ], ) - def edit_resource_chart( - self, info: Info, chart_input: ResourceChartInput - ) -> TypeResourceChart: + def edit_resource_chart(self, info: Info, chart_input: ResourceChartInput) -> TypeResourceChart: if not chart_input.chart_id: chart = ResourceChartDetails() else: @@ -316,7 +494,9 @@ def edit_resource_chart( _update_chart_fields(chart, chart_input, resource) return TypeResourceChart.from_django(chart) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def delete_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) @@ -325,7 +505,9 @@ def delete_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: except ResourceChartDetails.DoesNotExist as e: raise ValueError(f"Resource Chart with ID {chart_id} does not exist.") - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def publish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) @@ -335,7 +517,9 @@ def publish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: except ResourceChartDetails.DoesNotExist as e: raise ValueError(f"Resource Chart with ID {chart_id} does not exist.") - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartPermission] + ) def unpublish_resource_chart(self, info: Info, chart_id: uuid.UUID) -> bool: try: chart = ResourceChartDetails.objects.get(id=chart_id) diff --git a/api/schema/resource_schema.py b/api/schema/resource_schema.py index 06414f71..42f189b1 100644 --- a/api/schema/resource_schema.py +++ b/api/schema/resource_schema.py @@ -1,18 +1,18 @@ -import copy import uuid from enum import Enum # mypy: disable-error-code=operator -from typing import List, Optional +# mypy: disable-error-code=valid-type +from typing import Any, List, Optional import strawberry import strawberry_django import structlog +from django.db import transaction from django.db.models import QuerySet from strawberry.file_uploads import Upload from strawberry.types import Info -from api.managers.dvc_manager import DVCManager from api.models import ( Dataset, Resource, @@ -26,12 +26,45 @@ from api.types.type_resource import TypeResource from api.utils.constants import FORMAT_MAPPING from api.utils.data_indexing import index_resource_data +from api.utils.dataset_access import ChangeDatasetPartPermission, can_view_dataset from api.utils.file_utils import file_validation from api.utils.graphql_telemetry import trace_resolver logger = structlog.get_logger("dataspace.resource_schema") +def _dataset_from_arguments(kwargs: Any) -> Optional[Dataset]: + """The dataset a resource mutation acts on, from whichever argument it takes.""" + data = ( + kwargs.get("file_resource_input") + or kwargs.get("schema_update_input") + or kwargs.get("input") + ) + dataset_id = getattr(data, "dataset", None) + if dataset_id: + return Dataset.objects.filter(id=dataset_id).first() + resource_id = ( + kwargs.get("resource_id") + or getattr(data, "id", None) + or getattr(data, "resource", None) + or getattr(data, "resource_id", None) + ) + if not resource_id: + return None + resource = Resource.objects.select_related("dataset").filter(id=resource_id).first() + return resource.dataset if resource else None + + +class ChangeResourcePermission(ChangeDatasetPartPermission): + """Adding, changing or deleting a dataset's resources needs edit rights on the dataset.""" + + message = "You don't have permission to change this dataset's resources" + + def datasets(self, kwargs: Any) -> Optional[list]: + dataset = _dataset_from_arguments(kwargs) + return [dataset] if dataset else None + + @strawberry.input class CreateFileResourceInput: """Input type for creating a file resource.""" @@ -111,18 +144,22 @@ class Query: @strawberry_django.field @trace_resolver(name="get_dataset_resources", attributes={"component": "resource"}) - def dataset_resources( - self, info: Info, dataset_id: uuid.UUID - ) -> List[TypeResource]: + def dataset_resources(self, info: Info, dataset_id: uuid.UUID) -> List[TypeResource]: """Get resources for a dataset.""" - resources = Resource.objects.filter(dataset_id=dataset_id) + dataset = Dataset.objects.filter(id=dataset_id).first() + if not dataset or not can_view_dataset(info.context.user, dataset): + return [] + resources = Resource.objects.filter(dataset=dataset) return [TypeResource.from_django(resource) for resource in resources] @strawberry_django.field @trace_resolver(name="get_resource_by_id", attributes={"component": "resource"}) def resource_by_id(self, info: Info, resource_id: uuid.UUID) -> TypeResource: """Get a resource by ID.""" - resource = Resource.objects.get(id=resource_id) + resource = Resource.objects.select_related("dataset").get(id=resource_id) + if not can_view_dataset(info.context.user, resource.dataset): + # Same answer as a missing ID, so drafts' resources are not revealed. + raise Resource.DoesNotExist("Resource matching query does not exist.") return TypeResource.from_django(resource) @@ -133,8 +170,7 @@ def _validate_file_details_and_update_format(resource: Resource) -> None: raise ValueError("Resource has no file details") file = file_details.file - deep_copy_file = copy.deepcopy(file) - mime_type = file_validation(deep_copy_file, file, FORMAT_MAPPING) + mime_type = file_validation(file, file, FORMAT_MAPPING) if not mime_type: raise ValueError("Unsupported file format.") @@ -148,6 +184,12 @@ def _validate_file_details_and_update_format(resource: Resource) -> None: def _create_file_resource_schema(resource: Resource) -> None: """Create file resource schema.""" + # TODO: a failed indexing (the file saved, but its rows couldn't be loaded) still + # reports success, so the download serves the new file while previews, charts and + # the data API keep the old one. Pending a product decision: enforce the database's + # limits at upload (e.g. column names within 255 characters) and flag problem + # columns where people add column descriptions. See + # api/utils/data_indexing_architecture.md, Limitations. # Try to index CSV data if applicable data_table = index_resource_data(resource) @@ -162,23 +204,17 @@ def _reset_file_resource_schema(resource: Resource) -> None: data_table = index_resource_data(resource) -def _update_file_resource_schema( - resource: Resource, updated_schema: List[SchemaUpdate] -) -> None: +def _update_file_resource_schema(resource: Resource, updated_schema: List[SchemaUpdate]) -> None: """Update file resource schema and re-index if necessary.""" # Check if we need to re-index after schema update format_changes = False # Update schema fields - existing_schema: QuerySet[ResourceSchema] = ResourceSchema.objects.filter( - resource=resource - ) + existing_schema: QuerySet[ResourceSchema] = ResourceSchema.objects.filter(resource=resource) for schema in existing_schema: # type: ResourceSchema try: - schema_change = next( - item for item in updated_schema if item.id == str(schema.id) - ) + schema_change = next(item for item in updated_schema if item.id == str(schema.id)) # Check if format is changing, which might require re-indexing if schema.format != schema_change.format.value: format_changes = True @@ -188,9 +224,7 @@ def _update_file_resource_schema( schema.format = schema_change.format.value schema.save() - logger.info( - f"Updated schema field {schema.field_name} for resource {resource.id}" - ) + logger.info(f"Updated schema field {schema.field_name} for resource {resource.id}") except StopIteration: continue @@ -210,12 +244,8 @@ def _update_resource_preview_details( if file_resource_input.preview_details: # If preview_details already exists, update it if preview_details: - preview_details.is_all_entries = ( - file_resource_input.preview_details.is_all_entries - ) - preview_details.start_entry = ( - file_resource_input.preview_details.start_entry - ) + preview_details.is_all_entries = file_resource_input.preview_details.is_all_entries + preview_details.start_entry = file_resource_input.preview_details.start_entry preview_details.end_entry = file_resource_input.preview_details.end_entry preview_details.save() # Otherwise, create a new one @@ -235,6 +265,7 @@ class Mutation: @strawberry_django.mutation( handle_django_errors=False, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="created", @@ -260,9 +291,9 @@ def create_file_resources( raise ValueError(f"Dataset with ID {dataset_id} does not exist.") for file in file_resource_input.files: - resource = Resource.objects.create(name=file.name, dataset=dataset) + resource = Resource.objects.create(name=file.name, dataset=dataset) # type: ignore[attr-defined] ResourceFileDetails.objects.create( - file=file, size=file.size, resource=resource + file=file, size=file.size, resource=resource # type: ignore[attr-defined] ) _validate_file_details_and_update_format(resource) _create_file_resource_schema(resource) @@ -271,6 +302,7 @@ def create_file_resources( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="created", @@ -298,6 +330,7 @@ def create_file_resource( @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="updated", @@ -305,11 +338,7 @@ def create_file_resource( "resource_id": str(result.id), "resource_name": result.name, "updated_fields": { - "name": ( - file_resource_input.name - if file_resource_input.name - else None - ), + "name": (file_resource_input.name if file_resource_input.name else None), "description": ( file_resource_input.description if file_resource_input.description is not None @@ -317,8 +346,7 @@ def create_file_resource( ), "preview_enabled": file_resource_input.preview_enabled, "file_updated": file_resource_input.file is not None, - "preview_details_updated": file_resource_input.preview_details - is not None, + "preview_details_updated": file_resource_input.preview_details is not None, }, }, ) @@ -332,9 +360,7 @@ def update_file_resource( try: resource = Resource.objects.get(id=file_resource_input.id) except Resource.DoesNotExist as e: - raise ValueError( - f"Resource with ID {file_resource_input.id} does not exist." - ) + raise ValueError(f"Resource with ID {file_resource_input.id} does not exist.") if file_resource_input.name: resource.name = file_resource_input.name @@ -347,12 +373,12 @@ def update_file_resource( file_details = getattr(resource, "resourcefiledetails", None) if file_details: file_details.file = file_resource_input.file - file_details.size = file_resource_input.file.size + file_details.size = file_resource_input.file.size # type: ignore[attr-defined] file_details.save() else: ResourceFileDetails.objects.create( file=file_resource_input.file, - size=file_resource_input.file.size, + size=file_resource_input.file.size, # type: ignore[attr-defined] resource=resource, ) _validate_file_details_and_update_format(resource) @@ -363,10 +389,11 @@ def update_file_resource( return TypeResource.from_django(resource) - @strawberry_django.mutation(handle_django_errors=True) - @trace_resolver( - name="update_file_resource_schema", attributes={"component": "resource"} + @strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[ChangeResourcePermission], ) + @trace_resolver(name="update_file_resource_schema", attributes={"component": "resource"}) def update_file_resource_schema( self, info: Info, schema_update_input: SchemaUpdateInput ) -> TypeResource: @@ -374,20 +401,17 @@ def update_file_resource_schema( try: resource = Resource.objects.get(id=schema_update_input.resource) except Resource.DoesNotExist as e: - raise ValueError( - f"Resource with ID {schema_update_input.resource} does not exist." - ) + raise ValueError(f"Resource with ID {schema_update_input.resource} does not exist.") _update_file_resource_schema(resource, schema_update_input.updates) return TypeResource.from_django(resource) - @strawberry_django.mutation(handle_django_errors=True) - @trace_resolver( - name="reset_file_resource_schema", attributes={"component": "resource"} + @strawberry_django.mutation( + handle_django_errors=True, + permission_classes=[ChangeResourcePermission], ) - def reset_file_resource_schema( - self, info: Info, resource_id: uuid.UUID - ) -> TypeResource: + @trace_resolver(name="reset_file_resource_schema", attributes={"component": "resource"}) + def reset_file_resource_schema(self, info: Info, resource_id: uuid.UUID) -> TypeResource: """Reset file resource schema.""" try: resource = Resource.objects.get(id=resource_id) @@ -400,6 +424,7 @@ def reset_file_resource_schema( @strawberry_django.mutation( handle_django_errors=False, + permission_classes=[ChangeResourcePermission], extensions=[ TrackActivity( verb="deleted", @@ -421,6 +446,7 @@ def delete_file_resource(self, info: Info, resource_id: uuid.UUID) -> bool: @strawberry_django.mutation( handle_django_errors=True, + permission_classes=[ChangeResourcePermission], extensions=[ TrackModelActivity( verb="versioned", @@ -434,68 +460,34 @@ def delete_file_resource(self, info: Info, resource_id: uuid.UUID) -> bool: ], ) @trace_resolver(name="create_major_version", attributes={"component": "resource"}) - def create_major_version( - self, info: Info, input: CreateMajorVersionInput - ) -> TypeResource: + def create_major_version(self, info: Info, input: CreateMajorVersionInput) -> TypeResource: """Create a major version for a resource. This should be used when significant changes are made to the resource data structure, such as schema changes, column additions/removals, or other breaking changes. """ - import os - - from django.conf import settings - try: - # Get the resource - resource = Resource.objects.get(id=input.resource_id) + resource = Resource.objects.select_related("dataset").get(id=input.resource_id) except Resource.DoesNotExist: raise ValueError(f"Resource with ID {input.resource_id} does not exist") - # Get the latest version last_version = resource.versions.order_by("-created_at").first() - if not last_version: logger.warning( f"No previous version found for resource {resource.name}, creating initial version" ) new_version = "v1.0.0" else: - # Increment major version - new_version = _increment_version( - last_version.version_number, increment_type="major" - ) + new_version = _increment_version(last_version.version_number, increment_type="major") - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - - # Get the resource file path - file_path = resource.resourcefiledetails.file.path - - # Determine if file is large and should use chunking - file_size = ( - resource.resourcefiledetails.file.size - if hasattr(resource.resourcefiledetails.file, "size") - else os.path.getsize(file_path) - ) - use_chunked = file_size > 100 * 1024 * 1024 # 100MB threshold - - # Track with DVC - dvc_file = dvc.track_resource(file_path, chunked=use_chunked) - message = f"Major version update for resource: {resource.name} to version {new_version}" - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{resource.name}-{new_version}") - - # Create version record - ResourceVersion.objects.create( - resource=resource, - version_number=new_version, - change_description=input.description, - ) - - # Update resource version field - resource.version = new_version - resource.save(update_fields=["version"]) + with transaction.atomic(): + ResourceVersion.objects.create( + resource=resource, + version_number=new_version, + change_description=input.description, + ) + resource.version = new_version + resource.save(update_fields=["version"]) logger.info(f"Created major version {new_version} for resource {resource.name}") return TypeResource.from_django(resource) diff --git a/api/schema/resoure_chart_image_schema.py b/api/schema/resoure_chart_image_schema.py index 6e40728a..6447a58c 100644 --- a/api/schema/resoure_chart_image_schema.py +++ b/api/schema/resoure_chart_image_schema.py @@ -1,6 +1,6 @@ import datetime import uuid -from typing import List, Optional +from typing import Any, List, Optional import strawberry import strawberry_django @@ -10,6 +10,11 @@ from api.models import Dataset, ResourceChartImage from api.types.type_resource_chart_image import TypeResourceChartImage +from api.utils.dataset_access import ( + ChangeDatasetPartPermission, + can_view_dataset, + viewable_datasets_q, +) from api.utils.enums import ChartStatus @@ -34,6 +39,7 @@ class ResourceChartImageInputPartial: @strawberry.type(name="Query") class Query: + # Draft datasets' images are hidden by TypeResourceChartImage.get_queryset. resource_chart_images: list[TypeResourceChartImage] = strawberry_django.field() @strawberry_django.field(pagination=True) @@ -41,23 +47,51 @@ def dataset_resource_charts( self, info: Info, dataset_id: uuid.UUID ) -> List[TypeResourceChartImage]: """Get all resource chart images for a dataset.""" - images = ResourceChartImage.objects.filter(dataset_id=dataset_id) + images = ResourceChartImage.objects.filter(dataset_id=dataset_id).filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeResourceChartImage.from_django(image) for image in images] @strawberry_django.field - def resource_chart_image( - self, info: Info, image_id: uuid.UUID - ) -> TypeResourceChartImage: - try: - image = ResourceChartImage.objects.get(id=image_id) - return TypeResourceChartImage.from_django(image) - except ResourceChartImage.DoesNotExist as e: + def resource_chart_image(self, info: Info, image_id: uuid.UUID) -> TypeResourceChartImage: + image = ResourceChartImage.objects.select_related("dataset").filter(id=image_id).first() + # A hidden image gets the same answer as a missing one. + if not image or not image.dataset or not can_view_dataset(info.context.user, image.dataset): raise ValueError(f"Resource Chart Image with ID {image_id} does not exist.") + return TypeResourceChartImage.from_django(image) + + +class ChangeChartImagePermission(ChangeDatasetPartPermission): + """Chart images belong to a dataset; changing one needs edit rights on it. + + Moving an image to another dataset needs rights on both. + """ + + message = "You don't have permission to change this dataset's chart images" + + def datasets(self, kwargs: Any) -> Optional[list]: + data = kwargs.get("input") + found = [] + for dataset_id in (kwargs.get("dataset"), getattr(data, "dataset", None)): + if dataset_id: + dataset = Dataset.objects.filter(id=dataset_id).first() + if not dataset: + return None + found.append(dataset) + image_id = kwargs.get("resource_chart_image_id") or getattr(data, "id", None) + if image_id: + image = ResourceChartImage.objects.select_related("dataset").filter(id=image_id).first() + if not image or not image.dataset: + return None + found.append(image.dataset) + return found @strawberry.type class Mutation: - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) def create_resource_chart_image( self, info: Info, input: ResourceChartImageInput ) -> TypeResourceChartImage: @@ -68,14 +102,15 @@ def create_resource_chart_image( raise ValueError(f"Dataset with ID {input.dataset} does not exist.") now = datetime.datetime.now() image = ResourceChartImage.objects.create( - name=input.name - or f"New resource_chart_image {now.strftime('%d %b %Y - %H:%M:%S')}", + name=input.name or f"New resource_chart_image {now.strftime('%d %b %Y - %H:%M:%S')}", dataset=dataset_obj, image=input.image, ) return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=True) + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) def update_resource_chart_image( self, info: Info, input: ResourceChartImageInputPartial ) -> TypeResourceChartImage: @@ -96,10 +131,10 @@ def update_resource_chart_image( image.save() return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=True) - def add_resource_chart_image( - self, info: Info, dataset: uuid.UUID - ) -> TypeResourceChartImage: + @strawberry_django.mutation( + handle_django_errors=True, permission_classes=[ChangeChartImagePermission] + ) + def add_resource_chart_image(self, info: Info, dataset: uuid.UUID) -> TypeResourceChartImage: """Add a new resource chart image to a dataset.""" try: dataset_obj = Dataset.objects.get(id=dataset) @@ -113,10 +148,10 @@ def add_resource_chart_image( ) return TypeResourceChartImage.from_django(image) - @strawberry_django.mutation(handle_django_errors=False) - def publish_resource_chart_image( - self, info: Info, resource_chart_image_id: uuid.UUID - ) -> bool: + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) + def publish_resource_chart_image(self, info: Info, resource_chart_image_id: uuid.UUID) -> bool: try: image = ResourceChartImage.objects.get(id=resource_chart_image_id) image.status = ChartStatus.PUBLISHED @@ -127,7 +162,9 @@ def publish_resource_chart_image( f"Resource Chart Image with ID {resource_chart_image_id} does not exist." ) - @strawberry_django.mutation(handle_django_errors=False) + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) def unpublish_resource_chart_image( self, info: Info, resource_chart_image_id: uuid.UUID ) -> bool: @@ -141,10 +178,10 @@ def unpublish_resource_chart_image( f"Resource Chart Image with ID {resource_chart_image_id} does not exist." ) - @strawberry_django.mutation(handle_django_errors=False) - def delete_resource_chart_image( - self, info: Info, resource_chart_image_id: uuid.UUID - ) -> bool: + @strawberry_django.mutation( + handle_django_errors=False, permission_classes=[ChangeChartImagePermission] + ) + def delete_resource_chart_image(self, info: Info, resource_chart_image_id: uuid.UUID) -> bool: """Delete a resource chart image.""" try: image = ResourceChartImage.objects.get(id=resource_chart_image_id) diff --git a/api/schema/schema.py b/api/schema/schema.py index 5f919cfc..1ac6b59b 100644 --- a/api/schema/schema.py +++ b/api/schema/schema.py @@ -11,6 +11,7 @@ import api.schema.aimodel_schema import api.schema.collaborative_schema import api.schema.dataset_schema +import api.schema.event_schema import api.schema.dataspace_schema import api.schema.geography_schema import api.schema.metadata_schema @@ -32,6 +33,7 @@ from api.types.type_dataset import TypeTag from api.types.type_metadata import TypeMetadata from api.types.type_resource import TypeResource +from api.utils.dataset_access import viewable_datasets_q from api.utils.graphql_error_extension import ErrorFormatterExtension from api.utils.graphql_telemetry import TelemetryExtension, trace_resolver from authorization.graphql import Mutation as AuthMutation @@ -46,7 +48,9 @@ class DefaultQuery: @strawberry_django.field @trace_resolver(name="resources", attributes={"component": "default"}) def resources(self, info: Info) -> List[TypeResource]: - resources = Resource.objects.all() + resources = Resource.objects.filter( + viewable_datasets_q(info.context.user, prefix="dataset__") + ) return [TypeResource.from_django(resource) for resource in resources] @strawberry_django.field @@ -78,6 +82,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.user_schema.Query, api.schema.collaborative_schema.Query, api.schema.publication_schema.Query, + api.schema.event_schema.Query, api.schema.platform_import_schema.Query, AuthQuery, ), @@ -102,6 +107,7 @@ def tags(self, info: Info) -> List[TypeTag]: api.schema.tags_schema.Mutation, api.schema.collaborative_schema.Mutation, api.schema.publication_schema.Mutation, + api.schema.event_schema.Mutation, api.schema.platform_import_schema.Mutation, AuthMutation, ), diff --git a/api/schema/usecase_schema.py b/api/schema/usecase_schema.py index 0e6a8a01..7272b4aa 100644 --- a/api/schema/usecase_schema.py +++ b/api/schema/usecase_schema.py @@ -1,6 +1,7 @@ """Schema definitions for use cases.""" # mypy: disable-error-code=operator +# mypy: disable-error-code=valid-type import datetime import uuid @@ -9,6 +10,7 @@ import strawberry import strawberry_django from django.db import models +from django.db.models import QuerySet from strawberry import auto from strawberry.file_uploads import Upload from strawberry.types import Info @@ -35,7 +37,12 @@ ) from api.types.type_dataset import TypeDataset from api.types.type_organization import TypeOrganization -from api.types.type_usecase import TypeUseCase, UseCaseFilter, UseCaseOrder +from api.types.type_usecase import ( + TypeUseCase, + UseCaseFilter, + UseCaseOrder, + UseCaseResponse, +) from api.types.type_usecase_organization import ( TypeUseCaseOrganizationRelationship, relationship_type, @@ -46,6 +53,13 @@ UseCaseStatus, ) from api.utils.graphql_telemetry import trace_resolver +from api.utils.public_links import parse_public_id +from api.utils.qs_utils import ( + FilterSpec, + SortSpec, + get_pagination_window, + status_counts, +) from authorization.models import User from authorization.types import TypeUser @@ -94,6 +108,57 @@ class UseCaseInputPartial: completed_on: Optional[datetime.date] = None +# Fields a table client may filter or sort on. Explicit allowlists are what +# stop arbitrary columns (and relations) from becoming queryable. +USE_CASES_TABLE_FILTER_FIELDS = [ + "id", + "title", + "status", + "running_status", + "organization", + "organization__slug", + "sectors__id", + "sectors__slug", + "sectors__name", + "geographies__id", + "geographies__name", + "geographies__code", + "tags__value", + "sdgs__code", + "sdgs__slug", + "started_on", + "completed_on", + "created", + "modified", +] +USE_CASES_TABLE_SORT_FIELDS = [ + "title", + "created", + "modified", + "status", + "running_status", + "started_on", + "completed_on", +] + + +def _visible_use_cases(info: Info, include_public: Optional[bool] = False) -> QuerySet: + """The use cases a caller may list. Same rules as `useCases`.""" + user = info.context.user + organization = info.context.context.get("organization") + if organization: + queryset = UseCase.objects.filter(organization=organization) + elif user.is_superuser: + queryset = UseCase.objects.all() + elif user.is_authenticated: + queryset = UseCase.objects.filter(user=user) + else: + queryset = UseCase.objects.filter(status=UseCaseStatus.PUBLISHED) + if include_public: + queryset = queryset | UseCase.objects.filter(status=UseCaseStatus.PUBLISHED) + return queryset.distinct() + + @strawberry.type(name="Query") class Query: """Queries for use cases.""" @@ -137,6 +202,42 @@ def use_cases( return TypeUseCase.from_django_list(queryset) + @strawberry.field + @trace_resolver(name="use_cases_table", attributes={"component": "usecase"}) + def use_cases_table( + self, + info: Info, + filters: Optional[List[FilterSpec]] = None, + sort_options: Optional[List[SortSpec]] = None, + limit: int = 50, + offset: int = 0, + include_public: Optional[bool] = False, + ) -> UseCaseResponse: + """Use cases for a table UI: client-driven filters, sortable columns and + page controls backed by a real total count. + + Additive next to `useCases`, with the same visibility rules, plus + published use cases when `include_public` is set. Filters are `{field, + condition, value}` and sorts are `{field, direction}`, limited to the + allowlists above; anything else is a GraphQL error naming the allowed + fields. Default order: newest first. + """ + visible = _visible_use_cases(info, include_public) + window = get_pagination_window( + visible, + limit=limit, + offset=offset, + filters=filters, + filtering_allowed_fields=USE_CASES_TABLE_FILTER_FIELDS, + sort_options=sort_options or [SortSpec(field="created", direction="desc")], + sorting_allowed_fields=USE_CASES_TABLE_SORT_FIELDS, + ) + return UseCaseResponse( + data=TypeUseCase.from_django_list(window["data"]), + total_items_count=window["total_items_count"], + status_counts=status_counts(visible, filters, USE_CASES_TABLE_FILTER_FIELDS), + ) + @strawberry_django.field @trace_resolver(name="get_published_use_cases", attributes={"component": "usecase"}) def published_use_cases( @@ -173,18 +274,29 @@ def published_use_cases( return TypeUseCase.from_django_list(results) @strawberry_django.field - @trace_resolver( - name="get_datasets_by_use_case", attributes={"component": "usecase"} - ) + @trace_resolver(name="get_use_case_by_slug", attributes={"component": "usecase"}) + def use_case_by_slug(self, info: Info, public_id: str) -> Optional[TypeUseCase]: + """Published use case for a public page, found by its link. + + ``public_id`` takes the id itself or the whole link part (``urlSlug``); + only the id at the end is used, so a link with old title words still + works. Drafts, unknown ids and links without an id return null. + """ + parsed = parse_public_id(public_id) + if parsed is None: + return None + use_case = UseCase.objects.filter(public_id=parsed, status=UseCaseStatus.PUBLISHED).first() + return TypeUseCase.from_django(use_case) if use_case else None + + @strawberry_django.field + @trace_resolver(name="get_datasets_by_use_case", attributes={"component": "usecase"}) def dataset_by_use_case(self, info: Info, use_case_id: str) -> list[TypeDataset]: """Get datasets by use case.""" queryset = Dataset.objects.filter(usecase__id=use_case_id) return TypeDataset.from_django_list(queryset) @strawberry_django.field - @trace_resolver( - name="get_contributors_by_use_case", attributes={"component": "usecase"} - ) + @trace_resolver(name="get_contributors_by_use_case", attributes={"component": "usecase"}) def contributors_by_use_case(self, info: Info, use_case_id: str) -> list[TypeUser]: """Get contributors by use case.""" try: @@ -199,9 +311,7 @@ def contributors_by_use_case(self, info: Info, use_case_id: str) -> list[TypeUse def _update_usecase_tags(usecase: UseCase, tags: List[str]) -> None: usecase.tags.clear() for tag in tags: - usecase.tags.add( - Tag.objects.get_or_create(defaults={"value": tag}, value__iexact=tag)[0] - ) + usecase.tags.add(Tag.objects.get_or_create(defaults={"value": tag}, value__iexact=tag)[0]) usecase.save() @@ -245,9 +355,7 @@ def _add_update_usecase_metadata( metadata_field = Metadata.objects.get(id=metadata_input_item.id) if not metadata_field.enabled: _delete_existing_metadata(usecase) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} is not enabled." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} is not enabled.") uc_metadata = UseCaseMetadata( usecase=usecase, metadata_item=metadata_field, @@ -256,9 +364,7 @@ def _add_update_usecase_metadata( uc_metadata.save() except Metadata.DoesNotExist: _delete_existing_metadata(usecase) - raise ValueError( - f"Metadata with ID {metadata_input_item.id} does not exist." - ) + raise ValueError(f"Metadata with ID {metadata_input_item.id} does not exist.") @trace_resolver(name="delete_existing_metadata", attributes={"component": "usecase"}) @@ -334,10 +440,7 @@ def add_use_case(self, info: Info) -> TypeUseCase: else None ), "sectors": ( - [ - str(sector_id) - for sector_id in update_metadata_input.sectors - ] + [str(sector_id) for sector_id in update_metadata_input.sectors] if update_metadata_input.sectors else [] ), @@ -406,10 +509,7 @@ def update_use_case(self, info: Info, data: UseCaseInputPartial) -> TypeUseCase: usecase.started_on = data.started_on if data.completed_on is not None and data.completed_on is not strawberry.UNSET: usecase.completed_on = data.completed_on - if ( - data.running_status is not None - and data.running_status is not strawberry.UNSET - ): + if data.running_status is not None and data.running_status is not strawberry.UNSET: usecase.running_status = data.running_status if data.logo is not None and data.logo is not strawberry.UNSET: usecase.logo = data.logo @@ -421,9 +521,7 @@ def update_use_case(self, info: Info, data: UseCaseInputPartial) -> TypeUseCase: extensions=[ TrackActivity( verb="deleted", - get_data=lambda info, use_case_id, **kwargs: { - "usecase_id": use_case_id - }, + get_data=lambda info, use_case_id, **kwargs: {"usecase_id": use_case_id}, ) ], ) @@ -689,9 +787,7 @@ def remove_contributor_from_use_case( get_data=lambda result, use_case_id, user_ids, **kwargs: { "usecase_id": use_case_id, "usecase_title": result.title, - "updated_fields": { - "contributors": [str(user_id) for user_id in user_ids] - }, + "updated_fields": {"contributors": [str(user_id) for user_id in user_ids]}, }, ) ], diff --git a/api/services/event_service.py b/api/services/event_service.py new file mode 100644 index 00000000..6a070cc7 --- /dev/null +++ b/api/services/event_service.py @@ -0,0 +1,185 @@ +""" +event_service +────────────── +Domain helpers for the Event CRUD flow. GraphQL resolvers stay thin and call +these with plain values. +""" + +import datetime +from typing import Any, List, Optional + +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db import transaction +from django.db.models import QuerySet + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.luma import canonical_luma_src, validate_luma_embed + +_UNSET: Any = object() + + +def create_event( + *, + user: Any, + organization: Any, + name: str = "", + location: str = "", + start_date: Any = None, + end_date: Any = None, + luma_embed: Optional[str] = None, +) -> Event: + """Create a draft. A blank name is replaced with a generated one. + + An organization on the request owns the row; otherwise the user does. + A sent embed must already be a Luma embed URL. Dates, when both are sent, + must run forward. + """ + errors: dict[str, List[str]] = {} + embed = _normalize_embed(luma_embed, errors) + _check_date_order(start_date, end_date, errors) + if errors: + raise DjangoValidationError(errors) + + return Event.objects.create( + name=(name or "").strip() or _default_name(), + location=(location or "").strip(), + start_date=start_date, + end_date=end_date, + luma_embed=embed or {}, + organization=organization, + user=user, + status=EventStatus.DRAFT, + ) + + +def apply_event_update( + event: Event, + *, + name: Any = _UNSET, + location: Any = _UNSET, + start_date: Any = _UNSET, + end_date: Any = _UNSET, + luma_embed: Any = _UNSET, +) -> Event: + """Apply a partial update. Omitted fields are left untouched. + + A draft may be saved incomplete. A row that is already published cannot be + saved back into an incomplete state. + """ + errors: dict[str, List[str]] = {} + new_name = event.name if name is _UNSET else (name or "").strip() + new_location = event.location if location is _UNSET else (location or "").strip() + new_start = event.start_date if start_date is _UNSET else start_date + new_end = event.end_date if end_date is _UNSET else end_date + + new_embed = event.luma_embed or {} + if luma_embed is not _UNSET: + if luma_embed: + parsed = _normalize_embed(luma_embed, errors) + if parsed is not None: + new_embed = parsed + else: + new_embed = {} + + _check_date_order(new_start, new_end, errors) + if errors: + raise DjangoValidationError(errors) + + if event.status == EventStatus.PUBLISHED: + _require_complete( + name=new_name, + location=new_location, + start_date=new_start, + end_date=new_end, + luma_embed=new_embed, + ) + + with transaction.atomic(): + event.name = new_name + event.location = new_location + event.start_date = new_start + event.end_date = new_end + event.luma_embed = new_embed + event.save() + return event + + +def assert_ready_to_publish(event: Event) -> None: + """Reject a publish when the draft is still missing a required field.""" + _require_complete( + name=event.name, + location=event.location, + start_date=event.start_date, + end_date=event.end_date, + luma_embed=event.luma_embed or {}, + ) + + +def set_event_status(event: Event, status: EventStatus) -> Event: + """Flip an event's publish status and save it.""" + event.status = status + event.save() + return event + + +def get_scoped_events(*, user: Any, organization: Any) -> "QuerySet[Event, Event]": + """Return the events a caller may list. + + Organization context → that org's events; an authenticated individual → + their own; anonymous → published only. Ordered newest-first. + """ + if organization: + queryset = Event.objects.filter(organization=organization) + elif getattr(user, "is_superuser", False): + queryset = Event.objects.all() + elif getattr(user, "is_authenticated", False): + queryset = Event.objects.filter(user=user, organization__isnull=True) + else: + queryset = Event.objects.filter(status=EventStatus.PUBLISHED) + + return queryset.select_related("organization", "user").order_by("-modified") + + +def _require_complete( + *, + name: Optional[str], + location: Optional[str], + start_date: Any, + end_date: Any, + luma_embed: Any, +) -> None: + errors: dict[str, List[str]] = {} + if not name or not name.strip(): + errors["name"] = ["Name is required."] + if not location or not location.strip(): + errors["location"] = ["Location is required."] + if start_date is None: + errors["start_date"] = ["Start date is required."] + if end_date is None: + errors["end_date"] = ["End date is required."] + _check_date_order(start_date, end_date, errors) + src = luma_embed.get("src") if isinstance(luma_embed, dict) else "" + if not src or not canonical_luma_src(str(src)): + errors["luma_embed"] = ["A Luma embed is required."] + if errors: + raise DjangoValidationError(errors) + + +def _normalize_embed(value: Optional[str], errors: dict[str, List[str]]) -> Optional[dict]: + if value is None or not str(value).strip(): + return None + try: + return validate_luma_embed(value) + except DjangoValidationError as exc: + errors["luma_embed"] = [str(message) for message in exc.messages] + return None + + +def _check_date_order(start_date: Any, end_date: Any, errors: dict[str, List[str]]) -> None: + if start_date and end_date and end_date < start_date: + errors["end_date"] = ["End date must be on or after the start date."] + + +def _default_name() -> str: + return f"New event {datetime.datetime.now().strftime('%d %b %Y - %H:%M:%S')}" diff --git a/api/services/metadata_export/adapter.py b/api/services/metadata_export/adapter.py index 2899c71b..f0215760 100644 --- a/api/services/metadata_export/adapter.py +++ b/api/services/metadata_export/adapter.py @@ -25,7 +25,7 @@ def _public_base() -> str: def _api_base() -> str: - return getattr(settings, "PUBLIC_API_URL", "https://api.civicdataspace.in").rstrip("/") + return getattr(settings, "PUBLIC_API_URL", "https://api.datakeep.civicdays.in").rstrip("/") def _iso(dt: Any) -> Optional[str]: @@ -127,7 +127,7 @@ def dataset_to_record(dataset: Dataset) -> Dict[str, Any]: source = getattr(dataset, "source", None) org = dataset.organization user = dataset.user - landing = f"{_public_base()}/datasets/{dataset.slug}" + landing = f"{_public_base()}/datasets/{dataset.url_slug}" sectors = [vocab.concept(s.name, vocab.sector_uri(s.name)) for s in dataset.sectors.all()] geographies = [ diff --git a/api/services/metadata_export/contracts/licenses.csv b/api/services/metadata_export/contracts/licenses.csv index cba256f4..85fd1a65 100644 --- a/api/services/metadata_export/contracts/licenses.csv +++ b/api/services/metadata_export/contracts/licenses.csv @@ -1,61 +1,61 @@ key,label,code,uri,is_open,visible_on_dataspace,alt_codes GODL-India,Government Open Data License – India,GODL-India,https://civicdataspace.in/id/licence/godl-india,True,yes,dataspace_enum=GOVERNMENT_OPEN_DATA_LICENSE -CC-PDM-1.0,Creative Commons Public Domain Mark 1.0 Universal,CC-PDM-1.0,https://spdx.org/licenses/CC-PDM-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-PDM-1.0.json -CC-BY-1.0,Creative Commons Attribution 1.0 Generic,CC-BY-1.0,https://spdx.org/licenses/CC-BY-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-1.0.json -CC-BY-2.0,Creative Commons Attribution 2.0 Generic,CC-BY-2.0,https://spdx.org/licenses/CC-BY-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.0.json -CC-BY-2.5-AU,Creative Commons Attribution 2.5 Australia,CC-BY-2.5-AU,https://spdx.org/licenses/CC-BY-2.5-AU,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.5-AU.json -CC-BY-2.5,Creative Commons Attribution 2.5 Generic,CC-BY-2.5,https://spdx.org/licenses/CC-BY-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-2.5.json -CC-BY-3.0-AU,Creative Commons Attribution 3.0 Australia,CC-BY-3.0-AU,https://spdx.org/licenses/CC-BY-3.0-AU,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AU.json -CC-BY-3.0-AT,Creative Commons Attribution 3.0 Austria,CC-BY-3.0-AT,https://spdx.org/licenses/CC-BY-3.0-AT,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AT.json -CC-BY-3.0-DE,Creative Commons Attribution 3.0 Germany,CC-BY-3.0-DE,https://spdx.org/licenses/CC-BY-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-DE.json -CC-BY-3.0-IGO,Creative Commons Attribution 3.0 IGO,CC-BY-3.0-IGO,https://spdx.org/licenses/CC-BY-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-IGO.json -CC-BY-3.0-NL,Creative Commons Attribution 3.0 Netherlands,CC-BY-3.0-NL,https://spdx.org/licenses/CC-BY-3.0-NL,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-NL.json -CC-BY-3.0-US,Creative Commons Attribution 3.0 United States,CC-BY-3.0-US,https://spdx.org/licenses/CC-BY-3.0-US,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0-US.json -CC-BY-3.0,Creative Commons Attribution 3.0 Unported,CC-BY-3.0,https://spdx.org/licenses/CC-BY-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-3.0.json +CC-PDM-1.0,Creative Commons Public Domain Mark 1.0 Universal,CC-PDM-1.0,https://spdx.org/licenses/CC-PDM-1.0,False,no,dataspace_enum=CC_PDM_1_0|spdx_detail=https://spdx.org/licenses/CC-PDM-1.0.json +CC-BY-1.0,Creative Commons Attribution 1.0 Generic,CC-BY-1.0,https://spdx.org/licenses/CC-BY-1.0,False,no,dataspace_enum=CC_BY_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-1.0.json +CC-BY-2.0,Creative Commons Attribution 2.0 Generic,CC-BY-2.0,https://spdx.org/licenses/CC-BY-2.0,False,no,dataspace_enum=CC_BY_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-2.0.json +CC-BY-2.5-AU,Creative Commons Attribution 2.5 Australia,CC-BY-2.5-AU,https://spdx.org/licenses/CC-BY-2.5-AU,False,no,dataspace_enum=CC_BY_2_5_AU|spdx_detail=https://spdx.org/licenses/CC-BY-2.5-AU.json +CC-BY-2.5,Creative Commons Attribution 2.5 Generic,CC-BY-2.5,https://spdx.org/licenses/CC-BY-2.5,False,no,dataspace_enum=CC_BY_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-2.5.json +CC-BY-3.0-AU,Creative Commons Attribution 3.0 Australia,CC-BY-3.0-AU,https://spdx.org/licenses/CC-BY-3.0-AU,False,no,dataspace_enum=CC_BY_3_0_AU|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AU.json +CC-BY-3.0-AT,Creative Commons Attribution 3.0 Austria,CC-BY-3.0-AT,https://spdx.org/licenses/CC-BY-3.0-AT,False,no,dataspace_enum=CC_BY_3_0_AT|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-AT.json +CC-BY-3.0-DE,Creative Commons Attribution 3.0 Germany,CC-BY-3.0-DE,https://spdx.org/licenses/CC-BY-3.0-DE,False,no,dataspace_enum=CC_BY_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-DE.json +CC-BY-3.0-IGO,Creative Commons Attribution 3.0 IGO,CC-BY-3.0-IGO,https://spdx.org/licenses/CC-BY-3.0-IGO,False,no,dataspace_enum=CC_BY_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-IGO.json +CC-BY-3.0-NL,Creative Commons Attribution 3.0 Netherlands,CC-BY-3.0-NL,https://spdx.org/licenses/CC-BY-3.0-NL,False,no,dataspace_enum=CC_BY_3_0_NL|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-NL.json +CC-BY-3.0-US,Creative Commons Attribution 3.0 United States,CC-BY-3.0-US,https://spdx.org/licenses/CC-BY-3.0-US,False,no,dataspace_enum=CC_BY_3_0_US|spdx_detail=https://spdx.org/licenses/CC-BY-3.0-US.json +CC-BY-3.0,Creative Commons Attribution 3.0 Unported,CC-BY-3.0,https://spdx.org/licenses/CC-BY-3.0,False,no,dataspace_enum=CC_BY_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-3.0.json CC-BY-4.0,Creative Commons Attribution 4.0,CC-BY-4.0,http://creativecommons.org/licenses/by/4.0/,True,yes,authority_label=Creative Commons Attribution 4.0 International|dataspace_enum=CC_BY_4_0_ATTRIBUTION|spdx_detail=https://spdx.org/licenses/CC-BY-4.0.json -CC-BY-ND-1.0,Creative Commons Attribution No Derivatives 1.0 Generic,CC-BY-ND-1.0,https://spdx.org/licenses/CC-BY-ND-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-1.0.json -CC-BY-ND-2.0,Creative Commons Attribution No Derivatives 2.0 Generic,CC-BY-ND-2.0,https://spdx.org/licenses/CC-BY-ND-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.0.json -CC-BY-ND-2.5,Creative Commons Attribution No Derivatives 2.5 Generic,CC-BY-ND-2.5,https://spdx.org/licenses/CC-BY-ND-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.5.json -CC-BY-ND-3.0-DE,Creative Commons Attribution No Derivatives 3.0 Germany,CC-BY-ND-3.0-DE,https://spdx.org/licenses/CC-BY-ND-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0-DE.json -CC-BY-ND-3.0,Creative Commons Attribution No Derivatives 3.0 Unported,CC-BY-ND-3.0,https://spdx.org/licenses/CC-BY-ND-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0.json -CC-BY-ND-4.0,Creative Commons Attribution No Derivatives 4.0 International,CC-BY-ND-4.0,https://spdx.org/licenses/CC-BY-ND-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-ND-4.0.json -CC-BY-NC-1.0,Creative Commons Attribution Non Commercial 1.0 Generic,CC-BY-NC-1.0,https://spdx.org/licenses/CC-BY-NC-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-1.0.json -CC-BY-NC-2.0,Creative Commons Attribution Non Commercial 2.0 Generic,CC-BY-NC-2.0,https://spdx.org/licenses/CC-BY-NC-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.0.json -CC-BY-NC-2.5,Creative Commons Attribution Non Commercial 2.5 Generic,CC-BY-NC-2.5,https://spdx.org/licenses/CC-BY-NC-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.5.json -CC-BY-NC-3.0-DE,Creative Commons Attribution Non Commercial 3.0 Germany,CC-BY-NC-3.0-DE,https://spdx.org/licenses/CC-BY-NC-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-DE.json -CC-BY-NC-3.0-IGO,Creative Commons Attribution Non Commercial 3.0 IGO,CC-BY-NC-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-IGO.json -CC-BY-NC-3.0,Creative Commons Attribution Non Commercial 3.0 Unported,CC-BY-NC-3.0,https://spdx.org/licenses/CC-BY-NC-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0.json -CC-BY-NC-4.0,Creative Commons Attribution Non Commercial 4.0 International,CC-BY-NC-4.0,http://creativecommons.org/licenses/by-nc/4.0/,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-4.0.json -CC-BY-NC-ND-1.0,Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic,CC-BY-NC-ND-1.0,https://spdx.org/licenses/CC-BY-NC-ND-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-1.0.json -CC-BY-NC-ND-2.0,Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic,CC-BY-NC-ND-2.0,https://spdx.org/licenses/CC-BY-NC-ND-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.0.json -CC-BY-NC-ND-2.5,Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic,CC-BY-NC-ND-2.5,https://spdx.org/licenses/CC-BY-NC-ND-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.5.json -CC-BY-NC-ND-3.0-DE,Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany,CC-BY-NC-ND-3.0-DE,https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json -CC-BY-NC-ND-3.0-IGO,Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO,CC-BY-NC-ND-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json -CC-BY-NC-ND-3.0,Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported,CC-BY-NC-ND-3.0,https://spdx.org/licenses/CC-BY-NC-ND-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0.json -CC-BY-NC-ND-4.0,Creative Commons Attribution Non Commercial No Derivatives 4.0 International,CC-BY-NC-ND-4.0,https://spdx.org/licenses/CC-BY-NC-ND-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-4.0.json -CC-BY-NC-SA-1.0,Creative Commons Attribution Non Commercial Share Alike 1.0 Generic,CC-BY-NC-SA-1.0,https://spdx.org/licenses/CC-BY-NC-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-1.0.json -CC-BY-NC-SA-2.0-UK,Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales,CC-BY-NC-SA-2.0-UK,https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json -CC-BY-NC-SA-2.0,Creative Commons Attribution Non Commercial Share Alike 2.0 Generic,CC-BY-NC-SA-2.0,https://spdx.org/licenses/CC-BY-NC-SA-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0.json -CC-BY-NC-SA-2.0-DE,Creative Commons Attribution Non Commercial Share Alike 2.0 Germany,CC-BY-NC-SA-2.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.json -CC-BY-NC-SA-2.5,Creative Commons Attribution Non Commercial Share Alike 2.5 Generic,CC-BY-NC-SA-2.5,https://spdx.org/licenses/CC-BY-NC-SA-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.5.json -CC-BY-NC-SA-3.0-DE,Creative Commons Attribution Non Commercial Share Alike 3.0 Germany,CC-BY-NC-SA-3.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json -CC-BY-NC-SA-3.0-IGO,Creative Commons Attribution Non Commercial Share Alike 3.0 IGO,CC-BY-NC-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json -CC-BY-NC-SA-3.0,Creative Commons Attribution Non Commercial Share Alike 3.0 Unported,CC-BY-NC-SA-3.0,https://spdx.org/licenses/CC-BY-NC-SA-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0.json -CC-BY-NC-SA-4.0,Creative Commons Attribution Non Commercial Share Alike 4.0 International,CC-BY-NC-SA-4.0,https://spdx.org/licenses/CC-BY-NC-SA-4.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-4.0.json -CC-BY-SA-1.0,Creative Commons Attribution Share Alike 1.0 Generic,CC-BY-SA-1.0,https://spdx.org/licenses/CC-BY-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-1.0.json -CC-BY-SA-2.0-UK,Creative Commons Attribution Share Alike 2.0 England and Wales,CC-BY-SA-2.0-UK,https://spdx.org/licenses/CC-BY-SA-2.0-UK,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0-UK.json -CC-BY-SA-2.0,Creative Commons Attribution Share Alike 2.0 Generic,CC-BY-SA-2.0,https://spdx.org/licenses/CC-BY-SA-2.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0.json -CC-BY-SA-2.1-JP,Creative Commons Attribution Share Alike 2.1 Japan,CC-BY-SA-2.1-JP,https://spdx.org/licenses/CC-BY-SA-2.1-JP,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.1-JP.json -CC-BY-SA-2.5,Creative Commons Attribution Share Alike 2.5 Generic,CC-BY-SA-2.5,https://spdx.org/licenses/CC-BY-SA-2.5,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.5.json -CC-BY-SA-3.0-AT,Creative Commons Attribution Share Alike 3.0 Austria,CC-BY-SA-3.0-AT,https://spdx.org/licenses/CC-BY-SA-3.0-AT,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-AT.json -CC-BY-SA-3.0-DE,Creative Commons Attribution Share Alike 3.0 Germany,CC-BY-SA-3.0-DE,https://spdx.org/licenses/CC-BY-SA-3.0-DE,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-DE.json -CC-BY-SA-3.0,Creative Commons Attribution Share Alike 3.0 Unported,CC-BY-SA-3.0,https://spdx.org/licenses/CC-BY-SA-3.0,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0.json -CC-BY-NC-SA-2.0-FR,Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France,CC-BY-NC-SA-2.0-FR,https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json -CC-BY-SA-3.0-IGO,Creative Commons Attribution-ShareAlike 3.0 IGO,CC-BY-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-SA-3.0-IGO,False,no,spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-IGO.json +CC-BY-ND-1.0,Creative Commons Attribution No Derivatives 1.0 Generic,CC-BY-ND-1.0,https://spdx.org/licenses/CC-BY-ND-1.0,False,no,dataspace_enum=CC_BY_ND_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-1.0.json +CC-BY-ND-2.0,Creative Commons Attribution No Derivatives 2.0 Generic,CC-BY-ND-2.0,https://spdx.org/licenses/CC-BY-ND-2.0,False,no,dataspace_enum=CC_BY_ND_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.0.json +CC-BY-ND-2.5,Creative Commons Attribution No Derivatives 2.5 Generic,CC-BY-ND-2.5,https://spdx.org/licenses/CC-BY-ND-2.5,False,no,dataspace_enum=CC_BY_ND_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-ND-2.5.json +CC-BY-ND-3.0-DE,Creative Commons Attribution No Derivatives 3.0 Germany,CC-BY-ND-3.0-DE,https://spdx.org/licenses/CC-BY-ND-3.0-DE,False,no,dataspace_enum=CC_BY_ND_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0-DE.json +CC-BY-ND-3.0,Creative Commons Attribution No Derivatives 3.0 Unported,CC-BY-ND-3.0,https://spdx.org/licenses/CC-BY-ND-3.0,False,no,dataspace_enum=CC_BY_ND_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-3.0.json +CC-BY-ND-4.0,Creative Commons Attribution No Derivatives 4.0 International,CC-BY-ND-4.0,https://spdx.org/licenses/CC-BY-ND-4.0,False,no,dataspace_enum=CC_BY_ND_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-ND-4.0.json +CC-BY-NC-1.0,Creative Commons Attribution Non Commercial 1.0 Generic,CC-BY-NC-1.0,https://spdx.org/licenses/CC-BY-NC-1.0,False,no,dataspace_enum=CC_BY_NC_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-1.0.json +CC-BY-NC-2.0,Creative Commons Attribution Non Commercial 2.0 Generic,CC-BY-NC-2.0,https://spdx.org/licenses/CC-BY-NC-2.0,False,no,dataspace_enum=CC_BY_NC_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.0.json +CC-BY-NC-2.5,Creative Commons Attribution Non Commercial 2.5 Generic,CC-BY-NC-2.5,https://spdx.org/licenses/CC-BY-NC-2.5,False,no,dataspace_enum=CC_BY_NC_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-2.5.json +CC-BY-NC-3.0-DE,Creative Commons Attribution Non Commercial 3.0 Germany,CC-BY-NC-3.0-DE,https://spdx.org/licenses/CC-BY-NC-3.0-DE,False,no,dataspace_enum=CC_BY_NC_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-DE.json +CC-BY-NC-3.0-IGO,Creative Commons Attribution Non Commercial 3.0 IGO,CC-BY-NC-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0-IGO.json +CC-BY-NC-3.0,Creative Commons Attribution Non Commercial 3.0 Unported,CC-BY-NC-3.0,https://spdx.org/licenses/CC-BY-NC-3.0,False,no,dataspace_enum=CC_BY_NC_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-3.0.json +CC-BY-NC-4.0,Creative Commons Attribution Non Commercial 4.0 International,CC-BY-NC-4.0,http://creativecommons.org/licenses/by-nc/4.0/,False,no,dataspace_enum=CC_BY_NC_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-4.0.json +CC-BY-NC-ND-1.0,Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic,CC-BY-NC-ND-1.0,https://spdx.org/licenses/CC-BY-NC-ND-1.0,False,no,dataspace_enum=CC_BY_NC_ND_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-1.0.json +CC-BY-NC-ND-2.0,Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic,CC-BY-NC-ND-2.0,https://spdx.org/licenses/CC-BY-NC-ND-2.0,False,no,dataspace_enum=CC_BY_NC_ND_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.0.json +CC-BY-NC-ND-2.5,Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic,CC-BY-NC-ND-2.5,https://spdx.org/licenses/CC-BY-NC-ND-2.5,False,no,dataspace_enum=CC_BY_NC_ND_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-2.5.json +CC-BY-NC-ND-3.0-DE,Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany,CC-BY-NC-ND-3.0-DE,https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE,False,no,dataspace_enum=CC_BY_NC_ND_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-DE.json +CC-BY-NC-ND-3.0-IGO,Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO,CC-BY-NC-ND-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_ND_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0-IGO.json +CC-BY-NC-ND-3.0,Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported,CC-BY-NC-ND-3.0,https://spdx.org/licenses/CC-BY-NC-ND-3.0,False,no,dataspace_enum=CC_BY_NC_ND_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-3.0.json +CC-BY-NC-ND-4.0,Creative Commons Attribution Non Commercial No Derivatives 4.0 International,CC-BY-NC-ND-4.0,https://spdx.org/licenses/CC-BY-NC-ND-4.0,False,no,dataspace_enum=CC_BY_NC_ND_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-ND-4.0.json +CC-BY-NC-SA-1.0,Creative Commons Attribution Non Commercial Share Alike 1.0 Generic,CC-BY-NC-SA-1.0,https://spdx.org/licenses/CC-BY-NC-SA-1.0,False,no,dataspace_enum=CC_BY_NC_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-1.0.json +CC-BY-NC-SA-2.0-UK,Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales,CC-BY-NC-SA-2.0-UK,https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK,False,no,dataspace_enum=CC_BY_NC_SA_2_0_UK|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-UK.json +CC-BY-NC-SA-2.0,Creative Commons Attribution Non Commercial Share Alike 2.0 Generic,CC-BY-NC-SA-2.0,https://spdx.org/licenses/CC-BY-NC-SA-2.0,False,no,dataspace_enum=CC_BY_NC_SA_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0.json +CC-BY-NC-SA-2.0-DE,Creative Commons Attribution Non Commercial Share Alike 2.0 Germany,CC-BY-NC-SA-2.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE,False,no,dataspace_enum=CC_BY_NC_SA_2_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-DE.json +CC-BY-NC-SA-2.5,Creative Commons Attribution Non Commercial Share Alike 2.5 Generic,CC-BY-NC-SA-2.5,https://spdx.org/licenses/CC-BY-NC-SA-2.5,False,no,dataspace_enum=CC_BY_NC_SA_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.5.json +CC-BY-NC-SA-3.0-DE,Creative Commons Attribution Non Commercial Share Alike 3.0 Germany,CC-BY-NC-SA-3.0-DE,https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE,False,no,dataspace_enum=CC_BY_NC_SA_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-DE.json +CC-BY-NC-SA-3.0-IGO,Creative Commons Attribution Non Commercial Share Alike 3.0 IGO,CC-BY-NC-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO,False,no,dataspace_enum=CC_BY_NC_SA_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0-IGO.json +CC-BY-NC-SA-3.0,Creative Commons Attribution Non Commercial Share Alike 3.0 Unported,CC-BY-NC-SA-3.0,https://spdx.org/licenses/CC-BY-NC-SA-3.0,False,no,dataspace_enum=CC_BY_NC_SA_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-3.0.json +CC-BY-NC-SA-4.0,Creative Commons Attribution Non Commercial Share Alike 4.0 International,CC-BY-NC-SA-4.0,https://spdx.org/licenses/CC-BY-NC-SA-4.0,False,no,dataspace_enum=CC_BY_NC_SA_4_0|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-4.0.json +CC-BY-SA-1.0,Creative Commons Attribution Share Alike 1.0 Generic,CC-BY-SA-1.0,https://spdx.org/licenses/CC-BY-SA-1.0,False,no,dataspace_enum=CC_BY_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-1.0.json +CC-BY-SA-2.0-UK,Creative Commons Attribution Share Alike 2.0 England and Wales,CC-BY-SA-2.0-UK,https://spdx.org/licenses/CC-BY-SA-2.0-UK,False,no,dataspace_enum=CC_BY_SA_2_0_UK|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0-UK.json +CC-BY-SA-2.0,Creative Commons Attribution Share Alike 2.0 Generic,CC-BY-SA-2.0,https://spdx.org/licenses/CC-BY-SA-2.0,False,no,dataspace_enum=CC_BY_SA_2_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.0.json +CC-BY-SA-2.1-JP,Creative Commons Attribution Share Alike 2.1 Japan,CC-BY-SA-2.1-JP,https://spdx.org/licenses/CC-BY-SA-2.1-JP,False,no,dataspace_enum=CC_BY_SA_2_1_JP|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.1-JP.json +CC-BY-SA-2.5,Creative Commons Attribution Share Alike 2.5 Generic,CC-BY-SA-2.5,https://spdx.org/licenses/CC-BY-SA-2.5,False,no,dataspace_enum=CC_BY_SA_2_5|spdx_detail=https://spdx.org/licenses/CC-BY-SA-2.5.json +CC-BY-SA-3.0-AT,Creative Commons Attribution Share Alike 3.0 Austria,CC-BY-SA-3.0-AT,https://spdx.org/licenses/CC-BY-SA-3.0-AT,False,no,dataspace_enum=CC_BY_SA_3_0_AT|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-AT.json +CC-BY-SA-3.0-DE,Creative Commons Attribution Share Alike 3.0 Germany,CC-BY-SA-3.0-DE,https://spdx.org/licenses/CC-BY-SA-3.0-DE,False,no,dataspace_enum=CC_BY_SA_3_0_DE|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-DE.json +CC-BY-SA-3.0,Creative Commons Attribution Share Alike 3.0 Unported,CC-BY-SA-3.0,https://spdx.org/licenses/CC-BY-SA-3.0,False,no,dataspace_enum=CC_BY_SA_3_0|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0.json +CC-BY-NC-SA-2.0-FR,Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France,CC-BY-NC-SA-2.0-FR,https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR,False,no,dataspace_enum=CC_BY_NC_SA_2_0_FR|spdx_detail=https://spdx.org/licenses/CC-BY-NC-SA-2.0-FR.json +CC-BY-SA-3.0-IGO,Creative Commons Attribution-ShareAlike 3.0 IGO,CC-BY-SA-3.0-IGO,https://spdx.org/licenses/CC-BY-SA-3.0-IGO,False,no,dataspace_enum=CC_BY_SA_3_0_IGO|spdx_detail=https://spdx.org/licenses/CC-BY-SA-3.0-IGO.json CC-BY-SA-4.0,Creative Commons Attribution-ShareAlike 4.0,CC-BY-SA-4.0,http://creativecommons.org/licenses/by-sa/4.0/,True,yes,authority_label=Creative Commons Attribution Share Alike 4.0 International|dataspace_enum=CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE|spdx_detail=https://spdx.org/licenses/CC-BY-SA-4.0.json -CC-PDDC,Creative Commons Public Domain Dedication and Certification,CC-PDDC,https://spdx.org/licenses/CC-PDDC,False,no,spdx_detail=https://spdx.org/licenses/CC-PDDC.json -CC-SA-1.0,Creative Commons Share Alike 1.0 Generic,CC-SA-1.0,https://spdx.org/licenses/CC-SA-1.0,False,no,spdx_detail=https://spdx.org/licenses/CC-SA-1.0.json -CC0-1.0,Creative Commons Zero v1.0 Universal,CC0-1.0,https://creativecommons.org/publicdomain/zero/1.0/,True,yes,spdx_detail=https://spdx.org/licenses/CC0-1.0.json +CC-PDDC,Creative Commons Public Domain Dedication and Certification,CC-PDDC,https://spdx.org/licenses/CC-PDDC,False,no,dataspace_enum=CC_PDDC|spdx_detail=https://spdx.org/licenses/CC-PDDC.json +CC-SA-1.0,Creative Commons Share Alike 1.0 Generic,CC-SA-1.0,https://spdx.org/licenses/CC-SA-1.0,False,no,dataspace_enum=CC_SA_1_0|spdx_detail=https://spdx.org/licenses/CC-SA-1.0.json +CC0-1.0,Creative Commons Zero v1.0 Universal,CC0-1.0,https://creativecommons.org/publicdomain/zero/1.0/,True,yes,dataspace_enum=CC0_1_0|spdx_detail=https://spdx.org/licenses/CC0-1.0.json ODC-By-1.0,Open Data Commons Attribution License 1.0,ODC-By-1.0,https://opendatacommons.org/licenses/by/1-0/,True,yes,authority_label=Open Data Commons Attribution License v1.0|dataspace_enum=OPEN_DATA_COMMONS_BY_ATTRIBUTION|spdx_detail=https://spdx.org/licenses/ODC-By-1.0.json -PDDL-1.0,Open Data Commons Public Domain Dedication & License 1.0,PDDL-1.0,https://opendatacommons.org/licenses/pddl/1-0/,True,yes,spdx_detail=https://spdx.org/licenses/PDDL-1.0.json +PDDL-1.0,Open Data Commons Public Domain Dedication & License 1.0,PDDL-1.0,https://opendatacommons.org/licenses/pddl/1-0/,True,yes,dataspace_enum=PDDL_1_0|spdx_detail=https://spdx.org/licenses/PDDL-1.0.json ODbL-1.0,Open Database License 1.0,ODbL-1.0,https://opendatacommons.org/licenses/odbl/1-0/,True,yes,authority_label=Open Data Commons Open Database License v1.0|dataspace_enum=OPEN_DATABASE_LICENSE|spdx_detail=https://spdx.org/licenses/ODbL-1.0.json diff --git a/api/services/platform_import_service.py b/api/services/platform_import_service.py index 0889cc9e..131090e2 100644 --- a/api/services/platform_import_service.py +++ b/api/services/platform_import_service.py @@ -13,7 +13,6 @@ import structlog from django.db import transaction -from django.utils.text import slugify from api.models import ( Dataset, @@ -84,19 +83,6 @@ def find_existing_import( return source.dataset if source else None -def _unique_dataset_slug(title: str) -> str: - """Pick a free slug up front. Dataset.save() retries on a slug collision, - but only a handful of times; platform titles ("Iris", "Wine Reviews") - repeat across organisations far more often than timestamped native ones, - so choose a free slug before saving rather than rely on the retry.""" - base = slugify(title)[:240] or "imported-dataset" - slug, counter = base, 2 - while Dataset.objects.filter(slug=slug).exists(): - slug = f"{base}-{counter}" - counter += 1 - return slug - - PLATFORM_LABELS = {"HUGGINGFACE": "Hugging Face", "KAGGLE": "Kaggle", "GITHUB": "GitHub"} @@ -191,7 +177,6 @@ def _create_import( display_title = (title or "").strip()[:300] or info.title dataset = Dataset.objects.create( title=display_title, - slug=_unique_dataset_slug(display_title), description=info.description, user=user, organization=organization, diff --git a/api/services/platform_importers/base.py b/api/services/platform_importers/base.py index 61518bfb..0b15f8ff 100644 --- a/api/services/platform_importers/base.py +++ b/api/services/platform_importers/base.py @@ -93,8 +93,10 @@ def mapped_license(self) -> str: # --------------------------------------------------------------------------- # # Helpers shared by importers # --------------------------------------------------------------------------- # -# Platform license strings (lower-cased) -> DatasetLicense. Anything not listed -# falls back to CC-BY 4.0 and the raw string is kept on DatasetSource. +# Platform license strings (lower-cased) -> DatasetLicense, for names that are +# not an SPDX id. SPDX ids ("CC-BY-NC-4.0") match the enum directly in +# map_license. Anything else falls back to CC-BY 4.0 and the raw string is kept +# on DatasetSource. LICENSE_ALIASES: Dict[str, str] = { "cc-by-4.0": DatasetLicense.CC_BY_4_0_ATTRIBUTION, "cc-by": DatasetLicense.CC_BY_4_0_ATTRIBUTION, @@ -113,13 +115,25 @@ def mapped_license(self) -> str: "odc-odbl": DatasetLicense.OPEN_DATABASE_LICENSE, "database: open database, contents: database contents": DatasetLicense.OPEN_DATABASE_LICENSE, "database: open database, contents: © original authors": DatasetLicense.OPEN_DATABASE_LICENSE, + "cc0": DatasetLicense.CC0_1_0, + "cc0: public domain": DatasetLicense.CC0_1_0, + "pddl": DatasetLicense.PDDL_1_0, + "odc-pddl": DatasetLicense.PDDL_1_0, + "odc public domain dedication and licence (pddl)": DatasetLicense.PDDL_1_0, + "godl": DatasetLicense.GOVERNMENT_OPEN_DATA_LICENSE, + "godl-india": DatasetLicense.GOVERNMENT_OPEN_DATA_LICENSE, } def map_license(raw: str) -> str: """Map a platform license string onto DatasetLicense (default CC-BY-4.0).""" key = (raw or "").strip().lower() - return LICENSE_ALIASES.get(key, DatasetLicense.CC_BY_4_0_ATTRIBUTION) + if key in LICENSE_ALIASES: + return LICENSE_ALIASES[key] + spdx = re.sub(r"[^A-Z0-9]+", "_", key.upper()).strip("_") + if spdx in DatasetLicense.values: + return spdx + return DatasetLicense.CC_BY_4_0_ATTRIBUTION # Hard limits: the platforms' own maxima are well under these, so anything diff --git a/api/signals/__init__.py b/api/signals/__init__.py index 542aabaf..141dbf97 100644 --- a/api/signals/__init__.py +++ b/api/signals/__init__.py @@ -1,7 +1,9 @@ # Import signals to register them from api.signals import ( aimodel_signals, + data_table_signals, dataset_signals, + event_signals, publication_signals, usecase_signals, ) diff --git a/api/signals/data_table_signals.py b/api/signals/data_table_signals.py new file mode 100644 index 00000000..655ebec0 --- /dev/null +++ b/api/signals/data_table_signals.py @@ -0,0 +1,33 @@ +"""Drop a data table once the row that points at it is deleted. + +Re-uploads, resource deletes, dataset deletes (by cascade) and admin deletes +all delete the resource's ResourceDataTable row, so this one receiver frees the +storage for every one of them. The drop waits for the delete to commit, so a +rolled-back delete keeps its data, and it leaves a legacy table alone while +another row still shares it. + +Registered in api/signals/__init__.py (a test proves it is connected once the +app loads). Docs: ../utils/data_indexing_architecture.md +""" + +from functools import partial +from typing import Any + +from django.db import transaction +from django.db.models.signals import post_delete +from django.dispatch import receiver + +from api.models.Resource import ResourceDataTable +from api.utils.data_tables import drop_data_table_if_unshared + + +@receiver(post_delete, sender=ResourceDataTable) +def drop_table_after_row_delete( + sender: Any, instance: ResourceDataTable, using: str, **kwargs: Any +) -> None: + # Once the delete commits, drop the row's table (unless another row still uses it). + transaction.on_commit( + partial(drop_data_table_if_unshared, instance.table_name, instance.pk), + using=using, + robust=True, + ) diff --git a/api/signals/dataset_signals.py b/api/signals/dataset_signals.py index 3d4ac987..6f9f88ee 100644 --- a/api/signals/dataset_signals.py +++ b/api/signals/dataset_signals.py @@ -1,11 +1,9 @@ from typing import Any, Optional import structlog -from django.conf import settings from django.db.models.signals import pre_save from django.dispatch import receiver -from api.managers.dvc_manager import DVCManager from api.models.Dataset import Dataset from api.models.Resource import Resource, ResourceVersion, _increment_version from api.utils.enums import DatasetStatus @@ -77,16 +75,13 @@ def handle_dataset_publication(sender: Any, instance: Dataset, **kwargs: Any) -> # Get all resources for this dataset resources = instance.resources.all() - # Initialize DVC manager - dvc = DVCManager(settings.DVC_REPO_PATH) - # Increment version for each resource for resource in resources: # type: Resource try: # Get the latest version - last_version: Optional[ResourceVersion] = ( - resource.versions.order_by("-created_at").first() - ) + last_version: Optional[ResourceVersion] = resource.versions.order_by( + "-created_at" + ).first() if last_version: # Increment minor version (publication is a significant change) @@ -94,17 +89,6 @@ def handle_dataset_publication(sender: Any, instance: Dataset, **kwargs: Any) -> last_version.version_number, increment_type="minor" ) - # Get the resource file path - file_path: str = resource.resourcefiledetails.file.path - - # Track with DVC - dvc_file = dvc.track_resource(file_path) - message: str = ( - f"Publishing resource: {resource.name} to version {new_version}" - ) - dvc.commit_version(dvc_file, message) - dvc.tag_version(f"{resource.name}-{new_version}") - # Create version record ResourceVersion.objects.create( resource=resource, diff --git a/api/signals/event_signals.py b/api/signals/event_signals.py new file mode 100644 index 00000000..01a50b41 --- /dev/null +++ b/api/signals/event_signals.py @@ -0,0 +1,90 @@ +""" +event_signals +───────────── +Keep search honest: publishing an event adds it to the index, unpublishing or +deleting it removes it, so a draft is never searchable. Each index write is +followed by a search-cache bump so listings do not serve stale pages. +""" + +from typing import Any + +import structlog +from django.db.models.signals import post_delete, pre_save +from django.dispatch import receiver + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.search_cache import invalidate_search_cache +from search.documents import EventDocument + +logger = structlog.getLogger(__name__) + + +def _should_be_indexed(instance: Event) -> bool: + """An event belongs in search only while it is PUBLISHED.""" + return instance.status == EventStatus.PUBLISHED.value + + +@receiver(pre_save, sender=Event) +def handle_event_visibility(sender: Any, instance: Event, **kwargs: Any) -> None: + """Add / refresh / drop the search document as an event is published or not. + + New rows are handled by the django-elasticsearch-dsl signal processor. ES + errors are logged and swallowed — indexing is best-effort and a rebuild + reconciles it. + """ + if not instance.pk: + return + + try: + original = Event.objects.get(pk=instance.pk) + except Event.DoesNotExist: + return + + was_indexable = _should_be_indexed(original) + is_indexable = _should_be_indexed(instance) + + if was_indexable and is_indexable: + action = "update" + elif was_indexable and not is_indexable: + action = "delete" + elif not was_indexable and is_indexable: + action = "add" + else: + return + + try: + document = EventDocument.get(id=instance.id, ignore=404) + if action == "delete": + if document: + document.delete(refresh=True) + logger.info("event removed from search index", event_id=str(instance.id)) + else: + EventDocument().update(instance, refresh=True) + logger.info("event synced to search index", event_id=str(instance.id), action=action) + # After the index write, never before it: see invalidate_search_cache. + invalidate_search_cache() + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to sync event search document", + event_id=str(instance.id), + action=action, + error=str(exc), + ) + + +@receiver(post_delete, sender=Event) +def remove_event_document(sender: Any, instance: Event, **kwargs: Any) -> None: + """Drop the search document when an event is deleted.""" + try: + document = EventDocument.get(id=instance.id, ignore=404) + if document: + document.delete(refresh=True) + logger.info("deleted event removed from search index", event_id=str(instance.id)) + invalidate_search_cache() + except Exception as exc: # pragma: no cover - logging only + logger.error( + "failed to delete event search document", + event_id=str(instance.id), + error=str(exc), + ) diff --git a/api/types/type_aimodel.py b/api/types/type_aimodel.py index 7870d8cb..efc824ec 100644 --- a/api/types/type_aimodel.py +++ b/api/types/type_aimodel.py @@ -1,3 +1,5 @@ +# mypy: disable-error-code="valid-type" + """GraphQL types for AI Model.""" import uuid @@ -28,6 +30,7 @@ HFModelClass, PromptDomain, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser logger = structlog.get_logger("dataspace.type_aimodel") @@ -112,6 +115,7 @@ class TypeAIModel(BaseType): """GraphQL type for AI Model.""" id: int + public_id: str name: str display_name: str version: Optional[str] @@ -139,6 +143,10 @@ class TypeAIModel(BaseType): updated_at: datetime last_tested_at: Optional[datetime] + @strawberry.field(description="Link part for the AI model page: title words + publicId.") + def url_slug(self) -> str: + return cast(AIModel, self).url_slug + @strawberry.field def endpoints(self, info: Info) -> List[TypeModelEndpoint]: """Get endpoints for this AI model. @@ -332,3 +340,17 @@ def ai_model(self) -> Optional[TypeAIModel]: return TypeAIModel.from_django(django_instance.ai_model) except Exception: return None + + +@strawberry.type +class AIModelResponse: + """One page of a table listing (`aiModelsTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeAIModel] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_collaborative.py b/api/types/type_collaborative.py index 5fa74803..0fd2d963 100644 --- a/api/types/type_collaborative.py +++ b/api/types/type_collaborative.py @@ -1,4 +1,6 @@ -from typing import List, Optional +# mypy: disable-error-code="valid-type" + +from typing import List, Optional, cast import strawberry import strawberry_django @@ -29,6 +31,7 @@ OrganizationRelationshipType, PublicationStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser collaborative_status = strawberry.enum(CollaborativeStatus) # type: ignore @@ -70,9 +73,11 @@ class TypeCollaborative(BaseType): description="URL of the platform where this collaborative is published" ) - @strawberry.field( - description="Check if this collaborative is created by an individual user." - ) + @strawberry.field(description="Link part for the collaborative page: title words + publicId.") + def url_slug(self) -> str: + return cast(Collaborative, self).url_slug + + @strawberry.field(description="Check if this collaborative is created by an individual user.") def is_individual_collaborative(self) -> bool: """Check if this collaborative is created by an individual user.""" return self.organization is None @@ -123,9 +128,7 @@ def use_cases(self) -> Optional[List["TypeUseCase"]]: except Exception: return [] - @strawberry.field( - description="Get the count of datasets associated with this collaborative." - ) + @strawberry.field(description="Get the count of datasets associated with this collaborative.") def dataset_count(self: "TypeCollaborative", info: Info) -> int: """Get the count of datasets associated with this collaborative.""" try: @@ -133,9 +136,7 @@ def dataset_count(self: "TypeCollaborative", info: Info) -> int: except Exception: return 0 - @strawberry.field( - description="Get the count of use cases associated with this collaborative." - ) + @strawberry.field(description="Get the count of use cases associated with this collaborative.") def use_case_count(self: "TypeCollaborative", info: Info) -> int: """Get the count of use cases associated with this collaborative.""" try: @@ -198,9 +199,7 @@ def metadata(self) -> Optional[List["TypeCollaborativeMetadata"]]: except Exception: return [] - @strawberry.field( - description="Get contributors associated with this collaborative." - ) + @strawberry.field(description="Get contributors associated with this collaborative.") def contributors(self) -> Optional[List["TypeUser"]]: """Get contributors associated with this collaborative.""" try: @@ -226,9 +225,7 @@ def organization_relationships( except Exception: return [] - @strawberry.field( - description="Get supporting organizations for this collaborative." - ) + @strawberry.field(description="Get supporting organizations for this collaborative.") def supporting_organizations(self) -> Optional[List["TypeOrganization"]]: """Get supporting organizations for this collaborative.""" try: @@ -269,3 +266,17 @@ def partner_organizations(self) -> Optional[List["TypeOrganization"]]: return TypeOrganization.from_django_list(organizations) except Exception: return [] + + +@strawberry.type +class CollaborativeResponse: + """One page of a table listing (`collaborativesTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeCollaborative] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_collaborative_organization.py b/api/types/type_collaborative_organization.py index 662430f7..e6cfecfe 100644 --- a/api/types/type_collaborative_organization.py +++ b/api/types/type_collaborative_organization.py @@ -1,3 +1,5 @@ +# mypy: disable-error-code="valid-type" + """GraphQL type for UseCase-Organization relationship.""" from typing import Optional diff --git a/api/types/type_dataset.py b/api/types/type_dataset.py index e8765930..d53dcb74 100644 --- a/api/types/type_dataset.py +++ b/api/types/type_dataset.py @@ -57,6 +57,7 @@ class TypeDataset(BaseType): title: str description: Optional[str] slug: str + public_id: str status: dataset_status dataset_type: dataset_type_enum organization: Optional["TypeOrganization"] @@ -66,6 +67,11 @@ class TypeDataset(BaseType): download_count: int user: Optional["TypeUser"] + @strawberry.field + def url_slug(self) -> str: + """Link part for the dataset page: title words + public_id.""" + return cast(Dataset, self).url_slug + @strawberry.field def source(self) -> Optional["TypeDatasetSource"]: """Provenance for datasets imported from a third-party platform (else null).""" diff --git a/api/types/type_event.py b/api/types/type_event.py new file mode 100644 index 00000000..0df3c110 --- /dev/null +++ b/api/types/type_event.py @@ -0,0 +1,109 @@ +import uuid +from datetime import date, datetime +from typing import List, Optional, cast + +import strawberry +import strawberry_django +from strawberry.enum import EnumType + +from api.models import Event +from api.types.base_type import BaseType +from api.types.type_organization import TypeOrganization +from api.utils.enums import EventStatus +from api.utils.qs_utils import StatusCount +from authorization.types import TypeUser + +event_status: EnumType = strawberry.enum(EventStatus) # type: ignore + + +def _text(value: object) -> Optional[str]: + return value if isinstance(value, str) and value else None + + +@strawberry.type +class TypeLumaEmbed: + """Presentation attributes taken from a Luma iframe. ``src`` is always a Luma embed URL.""" + + src: str + width: Optional[str] + height: Optional[str] + style: Optional[str] + title: Optional[str] + allow: Optional[str] + allowfullscreen: bool + frameborder: Optional[str] + referrerpolicy: Optional[str] + loading: Optional[str] + + +@strawberry_django.filter(Event) +class EventFilter: + id: Optional[uuid.UUID] + status: Optional[event_status] + + +@strawberry_django.order(Event) +class EventOrder: + name: strawberry.auto + created: strawberry.auto + modified: strawberry.auto + start_date: strawberry.auto + + +@strawberry_django.type(Event, filters=EventFilter, pagination=True, order=EventOrder) # type: ignore +class TypeEvent(BaseType): + """A Luma event in the dashboard provider flow.""" + + id: uuid.UUID + name: str + public_id: str + location: str + start_date: Optional[date] + end_date: Optional[date] + status: event_status + created: datetime + modified: datetime + organization: Optional["TypeOrganization"] + user: Optional["TypeUser"] + + @strawberry.field + def luma_embed(self) -> Optional["TypeLumaEmbed"]: + """Whitelisted iframe attributes. Empty when the event has no embed yet.""" + raw = self.luma_embed if isinstance(getattr(self, "luma_embed", None), dict) else {} + src = raw.get("src") + if not isinstance(src, str) or not src: + return None + return TypeLumaEmbed( + src=src, + width=_text(raw.get("width")), + height=_text(raw.get("height")), + style=_text(raw.get("style")), + title=_text(raw.get("title")), + allow=_text(raw.get("allow")), + allowfullscreen=bool(raw.get("allowfullscreen")), + frameborder=_text(raw.get("frameborder")), + referrerpolicy=_text(raw.get("referrerpolicy")), + loading=_text(raw.get("loading")), + ) + + @strawberry.field + def url_slug(self) -> str: + """Public link part: title words plus publicId. Use it in event page URLs.""" + return cast(Event, self).url_slug + + @strawberry.field + def is_individual_event(self) -> bool: + """True when owned by an individual rather than an organization.""" + return self.organization is None and self.user is not None + + +@strawberry.type +class EventResponse: + """One page of ``eventsTable``: rows, the filtered total, and tab counts.""" + + data: List[TypeEvent] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter.", + ) diff --git a/api/types/type_metadata.py b/api/types/type_metadata.py index 45d37fae..132cc598 100644 --- a/api/types/type_metadata.py +++ b/api/types/type_metadata.py @@ -1,3 +1,5 @@ +# mypy: disable-error-code="valid-type" + from enum import Enum from typing import List, Optional @@ -25,7 +27,7 @@ class ValidatorType(Enum): class MetadataFilter: """Filter for metadata.""" - model: MetadataModels + model: metadata_models enabled: bool @@ -35,7 +37,7 @@ class TypeMetadata(BaseType): id: auto label: auto - model: MetadataModels + model: metadata_models type: auto enabled: auto validator: List[ValidatorType] diff --git a/api/types/type_publication.py b/api/types/type_publication.py index ee2781d1..b13dbd70 100644 --- a/api/types/type_publication.py +++ b/api/types/type_publication.py @@ -25,6 +25,7 @@ PublicationStatus, UseCaseStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser @@ -128,6 +129,7 @@ class TypePublication(BaseType): title: str description: Optional[str] slug: str + public_id: str status: publication_status authors: List[str] publication_date: Optional[date] @@ -140,6 +142,10 @@ class TypePublication(BaseType): user: Optional["TypeUser"] resource_type: Optional["TypeResourceType"] + @strawberry.field(description="Link part for the publication page: title words + publicId.") + def url_slug(self) -> str: + return cast(Publication, self).url_slug + @strawberry.field def sectors(self, info: Info) -> List["TypeSector"]: """Sectors tagged on this resource.""" @@ -236,3 +242,17 @@ def linked_count(self, info: Info) -> int: return usecases + collabs except (AttributeError, Publication.DoesNotExist): return 0 + + +@strawberry.type +class PublicationResponse: + """One page of a table listing (`publicationsTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypePublication] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_resource_chart.py b/api/types/type_resource_chart.py index 79fe331e..4bdc4ae1 100644 --- a/api/types/type_resource_chart.py +++ b/api/types/type_resource_chart.py @@ -1,5 +1,9 @@ +# mypy: disable-error-code="valid-type" + import json + from pyecharts.globals import CurrentConfig + CurrentConfig.ONLINE_HOST = "file:///code/" import uuid from datetime import datetime @@ -17,8 +21,10 @@ from api.types.charts.chart_registry import CHART_REGISTRY from api.types.type_dataset import TypeDataset from api.types.type_resource import TypeResource, TypeResourceSchema +from api.types.type_resource_chart_image import TypeResourceChartImage from api.utils.enums import ChartStatus, ChartTypes from api.utils.file_utils import load_csv +from api.utils.qs_utils import StatusCount logger = structlog.get_logger(__name__) T = TypeVar("T", bound="TypeResourceChart") @@ -39,8 +45,10 @@ def chart_base(chart_details: ResourceChartDetails) -> Optional[Chart]: logger.error("invalid resource format") return None - data = load_csv(file_details.file.path) - except (AttributeError, FileNotFoundError): + # Charts read the indexed data table, not the file; only require that it exists. + if not file_details.file.storage.exists(file_details.file.name): + return None + except AttributeError: return None chart_class = CHART_REGISTRY.get(chart_details.chart_type) @@ -316,9 +324,7 @@ def chart_options(self) -> Optional[ChartOptionsType]: point_size=options_dict.get("point_size"), # Geospatial Map Chart options geospatial_field=( - ensure_type( - options_dict.get("geospatial_field"), TypeResourceSchema - ) + ensure_type(options_dict.get("geospatial_field"), TypeResourceSchema) if options_dict.get("geospatial_field") else None ), @@ -334,9 +340,7 @@ def chart_filters(self) -> List[FilterType]: return [ FilterType( column=( - ensure_type(f["column"], TypeResourceSchema) - if f.get("column") - else None + ensure_type(f["column"], TypeResourceSchema) if f.get("column") else None ), operator=f["operator"], value=f["value"], @@ -354,9 +358,7 @@ def chart(self, info: Info) -> Optional[ChartConfig]: return None # Convert chart to JSON-serializable format - chart_options = ( - chart_instance.dump_options_with_quotes() if chart_instance else None - ) + chart_options = chart_instance.dump_options_with_quotes() if chart_instance else None if not chart_options: return None @@ -366,3 +368,17 @@ def chart(self, info: Info) -> Optional[ChartConfig]: height=chart_instance.height, renderer=chart_instance.renderer, ) + + +@strawberry.type +class ChartResponse: + """One page of the charts table (`chartsTable`): built charts and chart + images in one list, the total after filters, and per-status counts.""" + + data: List[Union[TypeResourceChartImage, TypeResourceChart]] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels do not change when switching tabs.", + ) diff --git a/api/types/type_resource_chart_image.py b/api/types/type_resource_chart_image.py index d0cf6acb..92c491d5 100644 --- a/api/types/type_resource_chart_image.py +++ b/api/types/type_resource_chart_image.py @@ -1,12 +1,17 @@ -from typing import Optional +# mypy: disable-error-code="valid-type" + +from typing import Any, Optional import strawberry import strawberry_django +from django.db.models import QuerySet from strawberry import auto +from strawberry.types import Info from api.models import ResourceChartImage from api.types.base_type import BaseType from api.types.type_dataset import TypeDataset +from api.utils.dataset_access import viewable_datasets_q from api.utils.enums import ChartStatus @@ -30,3 +35,8 @@ class TypeResourceChartImage(BaseType): description: Optional[str] = "" dataset: Optional[TypeDataset] = None status: ChartStatusEnum = ChartStatusEnum.DRAFT + + @classmethod + def get_queryset(cls, queryset: QuerySet, info: Info, **kwargs: Any) -> QuerySet: + """Only images of datasets the caller may see.""" + return queryset.filter(viewable_datasets_q(info.context.user, prefix="dataset__")) diff --git a/api/types/type_usecase.py b/api/types/type_usecase.py index 48d16400..6540daf4 100644 --- a/api/types/type_usecase.py +++ b/api/types/type_usecase.py @@ -1,4 +1,6 @@ -from typing import List, Optional +# mypy: disable-error-code="valid-type" + +from typing import List, Optional, cast import strawberry import strawberry_django @@ -28,6 +30,7 @@ PublicationStatus, UseCaseStatus, ) +from api.utils.qs_utils import StatusCount from authorization.types import TypeUser use_case_status = strawberry.enum(UseCaseStatus) # type: ignore @@ -69,9 +72,11 @@ class TypeUseCase(BaseType): description="URL of the platform where this use case is published" ) - @strawberry.field( - description="Check if this use case is created by an individual user." - ) + @strawberry.field(description="Link part for the use case page: title words + publicId.") + def url_slug(self) -> str: + return cast(UseCase, self).url_slug + + @strawberry.field(description="Check if this use case is created by an individual user.") def is_individual_usecase(self) -> bool: """Check if this use case is created by an individual user.""" return self.organization is None @@ -88,9 +93,7 @@ def datasets(self) -> Optional[List["TypeDataset"]]: except Exception: return [] - @strawberry.field( - description="Get the count of datasets associated with this use case." - ) + @strawberry.field(description="Get the count of datasets associated with this use case.") def dataset_count(self: "TypeUseCase", info: Info) -> int: """Get the count of datasets associated with this use case.""" try: @@ -185,9 +188,7 @@ def contributors(self) -> Optional[List["TypeUser"]]: except Exception: return [] - @strawberry.field( - description="Get organization relationships associated with this use case." - ) + @strawberry.field(description="Get organization relationships associated with this use case.") def organization_relationships( self, ) -> Optional[List["TypeUseCaseOrganizationRelationship"]]: @@ -234,9 +235,7 @@ def partner_organizations(self) -> Optional[List["TypeOrganization"]]: except Exception: return [] - @strawberry.field( - description="Get Usecase dashboard associated with this use case." - ) + @strawberry.field(description="Get Usecase dashboard associated with this use case.") def usecase_dashboard(self) -> Optional[List["TypeUseCaseDashboard"]]: """Get Usecase dashboard associated with this use case.""" try: @@ -246,3 +245,17 @@ def usecase_dashboard(self) -> Optional[List["TypeUseCaseDashboard"]]: return TypeUseCaseDashboard.from_django_list(usecase_dashboards) except Exception: return [] + + +@strawberry.type +class UseCaseResponse: + """One page of a table listing (`useCasesTable`): rows, the total after + filters, and per-status counts for tab labels.""" + + data: List[TypeUseCase] + total_items_count: int + status_counts: List[StatusCount] = strawberry.field( + default_factory=list, + description="Counts per status with the same filters applied, except any status filter, " + "so tab labels stay stable when switching tabs.", + ) diff --git a/api/types/type_usecase_organization.py b/api/types/type_usecase_organization.py index 2430dc9e..2f7fd2f2 100644 --- a/api/types/type_usecase_organization.py +++ b/api/types/type_usecase_organization.py @@ -1,3 +1,5 @@ +# mypy: disable-error-code="valid-type" + """GraphQL type for UseCase-Organization relationship.""" from typing import Optional diff --git a/api/urls.py b/api/urls.py index f188176b..f2170425 100644 --- a/api/urls.py +++ b/api/urls.py @@ -11,6 +11,7 @@ aimodel_execution, auditor, auth, + dataset_data, download, generate_dynamic_chart, metadata_export, @@ -18,6 +19,7 @@ search_aimodel, search_collaborative, search_dataset, + search_event, search_publication, search_publisher, search_unified, @@ -63,6 +65,7 @@ search_collaborative.SearchCollaborative.as_view(), name="search_collaborative", ), + path("search/event/", search_event.SearchEvent.as_view(), name="search_event"), path("search/unified/", search_unified.UnifiedSearch.as_view(), name="search_unified"), path("search/publisher/", search_publisher.SearchPublisher.as_view(), name="search_publisher"), path( @@ -85,6 +88,21 @@ trending_datasets.TrendingDatasets.as_view(), name="trending_datasets", ), + path( + "resources//data/", + dataset_data.ResourceDataView.as_view(), + name="resource_data", + ), + path( + "datasets//data/", + dataset_data.DatasetDataView.as_view(), + name="dataset_data", + ), + path( + "datasets//prompts/", + dataset_data.PromptDatasetDataView.as_view(), + name="prompt_dataset_data", + ), path( "datasets//export/", metadata_export.metadata_export, diff --git a/api/utils/data_indexing.py b/api/utils/data_indexing.py index e93ac56f..22159628 100644 --- a/api/utils/data_indexing.py +++ b/api/utils/data_indexing.py @@ -1,330 +1,153 @@ -from typing import Any, Dict, Generator, Optional +"""Copying tabular uploads into the data database, and reading them back. + +The upload flow is ``index_resource_data`` below; its helpers live in +./data_tables.py. The rest of this module reads the tables (row count, +preview, column list, the data API's filtered fetch, chart queries); every +reader looks the table up from the resource's ResourceDataTable row. + +Docs: ./data_indexing_architecture.md +""" + +from typing import Any, Dict, List, Optional, Tuple import pandas as pd import structlog from django.db import connections, transaction from django.db.utils import ProgrammingError +from django.utils import timezone +from psycopg2 import sql as pg_sql # type: ignore[import-untyped] from api.models.Resource import Resource, ResourceDataTable from api.models.ResourceSchema import ResourceSchema from api.types.type_preview_data import PreviewData -from api.utils.file_utils import load_tabular_data +from api.utils.data_tables import ( # noqa: F401 — DATA_DB, INDEXED_FORMATS, get_sql_type re-exported + DATA_DB, + INDEXED_FORMATS, + LOAD_TIMEOUT_SECONDS, + NewerFileSaved, + build_data_table, + data_table_row_exists, + describe_data_table, + drop_unswitched_table, + error_fields, + get_sql_type, + has_indexable_file, + is_inside_transaction, + load_resource_frame, + lock_resource, + new_data_table_row, + rebuild_resource_schema, + retire_data_table_row, + save_data_table_row, + stored_file_identity, +) +from api.utils.file_utils import local_copy logger = structlog.get_logger("dataspace.data_indexing") -# Use a separate database for data tables -DATA_DB = "data_db" # This should match the connection name in settings.py - -# Formats indexed into ResourceDataTable (queryable via get_row_count) -INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} # Formats counted by parsing the file directly (not indexed into DB) FILE_COUNT_FORMATS = {"yml", "yaml", "xml"} +# Allowed comparison operators for column-based filtering on indexed data. +# Maps operator suffix -> (sql_template_with_{ph}_placeholder, value_transformer) +_FILTER_OPERATORS: Dict[str, Tuple[str, Any]] = { + "eq": ("= %s", lambda v: v), + "ne": ("<> %s", lambda v: v), + "gt": ("> %s", lambda v: v), + "gte": (">= %s", lambda v: v), + "lt": ("< %s", lambda v: v), + "lte": ("<= %s", lambda v: v), + "in": ("= ANY(%s)", lambda v: list(v) if not isinstance(v, list) else v), + "nin": ("<> ALL(%s)", lambda v: list(v) if not isinstance(v, list) else v), + "contains": ("LIKE %s", lambda v: f"%{v}%"), + "icontains": ("ILIKE %s", lambda v: f"%{v}%"), + "startswith": ("LIKE %s", lambda v: f"{v}%"), + "istartswith": ("ILIKE %s", lambda v: f"{v}%"), + "endswith": ("LIKE %s", lambda v: f"%{v}"), + "iendswith": ("ILIKE %s", lambda v: f"%{v}"), + "isnull": ("IS NULL", None), # value ignored + "notnull": ("IS NOT NULL", None), +} + + +def index_resource_data( + resource: Resource, load_timeout: Optional[int] = LOAD_TIMEOUT_SECONDS +) -> Optional[ResourceDataTable]: + """Copy a resource's tabular file into a table of its own in the data database. + + Every upload gets a new table, named from its new ResourceDataTable row. + The table is built in full first; then one main-database commit points the + resource at it and rebuilds the column list. The old table is dropped after + that commit by the row's delete receiver. Readers see the complete old data + until the commit, then the complete new data. If anything fails, the old + data stays as it was and this build's table is dropped. + + Must be called outside any transaction, so "committed" really means + committed. ``load_timeout`` limits reading the file (seconds; None for no + limit, as the migration command uses). Returns the new ResourceDataTable, + or None when the resource was not switched over. + """ + resource_id = str(resource.id) -def get_sql_type(pandas_dtype: str) -> str: - """Convert pandas dtype to SQL type.""" - if "int" in pandas_dtype: - return "BIGINT" - elif "float" in pandas_dtype: - return "DOUBLE PRECISION" - elif "datetime" in pandas_dtype: - return "TIMESTAMP" - elif "bool" in pandas_dtype: - return "BOOLEAN" - else: - return "TEXT" - - -def create_table_for_resource(resource: Resource, df: pd.DataFrame) -> Optional[ResourceDataTable]: - """Create a database table for the resource data and index it.""" - try: - # Create ResourceDataTable entry first to get the table name - data_table = ResourceDataTable.objects.create(resource=resource) - table_name = data_table.table_name - - # Get column types - column_types = {col: get_sql_type(str(df[col].dtype)) for col in df.columns} - - # Create table in the data database - with connections[DATA_DB].cursor() as cursor: - # Drop table if exists - cursor.execute(f'DROP TABLE IF EXISTS "{table_name}"') - - # Create table with proper column types - columns = [f'"{col}" {dtype}' for col, dtype in column_types.items()] - create_table_sql = f'CREATE TABLE "{table_name}" ({", ".join(columns)})' - cursor.execute(create_table_sql) - - # Create temporary table for COPY - temp_table = f"temp_{table_name}" - cursor.execute(f'CREATE TEMP TABLE "{temp_table}" (LIKE "{table_name}")') - - # Copy data to temp table - quoted_columns = [f'"{col}"' for col in df.columns] - # Use StringIO to create a file-like object for the CSV data - from io import StringIO - - csv_data = StringIO() - df.to_csv(csv_data, index=False, header=False) - csv_data.seek(0) - - copy_sql = f'COPY "{temp_table}" ({",".join(quoted_columns)}) FROM STDIN WITH CSV' - cursor.copy_expert(copy_sql, csv_data) - - # Insert from temp to main table with validation - cursor.execute(f'INSERT INTO "{table_name}" SELECT * FROM "{temp_table}"') - cursor.execute(f'DROP TABLE "{temp_table}"') - - return data_table - - except Exception as e: - logger.error(f"Error creating table for resource {resource.id}: {str(e)}") - if "data_table" in locals(): - data_table.delete() + # Refuse to run inside someone else's transaction: the switch would only release a savepoint. + if is_inside_transaction(): + logger.error("data_index_refused_inside_transaction", resource_id=resource_id) return None + # Only supported tabular files (csv, xlsx, parquet, ...) get a table. + if not has_indexable_file(resource): + return None -def index_resource_data(resource: Resource) -> Optional[ResourceDataTable]: - """Index a resource's CSV data into a database table. - - This function handles the indexing process gracefully, catching and logging - specific errors at each stage of the process without crashing. - - Args: - resource: The Resource object to index - - Returns: - Optional[ResourceDataTable]: The created data table or None if indexing failed - """ - resource_id = getattr(resource, "id", "unknown") + # Read the file into a frame, remembering exactly which stored file it was. + loaded = load_resource_frame(resource, timeout=load_timeout) + if loaded is None: + return None + # This upload's own row, and with it its own table name. + new_row = new_data_table_row(resource) + switched = False try: - # Check if resource is a supported tabular file - try: - file_details = resource.resourcefiledetails - if not file_details: - logger.info(f"Resource {resource_id} has no file details, skipping indexing") - return None - except Exception as e: - logger.error(f"Failed to access file details for resource {resource_id}: {str(e)}") - return None - - # Check file format - try: - format = file_details.format.lower() - supported_formats = [ - "csv", - "xls", - "xlsx", - "ods", - "parquet", - "feather", - "json", - "tsv", - ] - if format not in supported_formats: - logger.info( - f"Resource {resource_id} has unsupported format: {format}, skipping indexing" - ) - return None - except Exception as e: - logger.error(f"Failed to determine format for resource {resource_id}: {str(e)}") - return None - - # Load tabular data with timeout protection - try: - import signal - from contextlib import contextmanager - - @contextmanager - def timeout(seconds: int) -> Generator[None, None, None]: - def handler(signum: int, frame: Any) -> None: - raise TimeoutError(f"Loading data timed out after {seconds} seconds") - - # Set the timeout handler - original_handler = signal.getsignal(signal.SIGALRM) - signal.signal(signal.SIGALRM, handler) - signal.alarm(seconds) - try: - yield - finally: - signal.alarm(0) - signal.signal(signal.SIGALRM, original_handler) - - # Apply timeout only on Unix-based systems where signal.SIGALRM is available - try: - with timeout(60): # 60 second timeout for loading data - df = load_tabular_data(file_details.file.path, format) - except TimeoutError as te: - logger.error(f"Timeout while loading data for resource {resource_id}: {str(te)}") - return None - except Exception: - # Fallback without timeout if signal.SIGALRM is not available (e.g., on Windows) - df = load_tabular_data(file_details.file.path, format) - - if df is None: - logger.error( - f"Failed to load data for resource {resource_id}: Data loading returned None" - ) - return None - if df.empty: - logger.info(f"Resource {resource_id} has empty data, skipping indexing") - return None - - # Check for problematic column names and sanitize them - try: - # Replace problematic characters in column names - # Create a new list of sanitized column names - sanitized_columns = [ - str(col).replace('"', "").replace("\n", " ").replace("\t", " ") - for col in df.columns - ] - # Assign to df.columns using pandas Index constructor - df.columns = pd.Index(sanitized_columns) - - # Check for duplicate column names after sanitization - if len(df.columns) != len(set(df.columns)): - # Handle duplicate columns by adding suffixes - from collections import Counter - - col_counter = Counter(df.columns) - for col, count in col_counter.items(): - if count > 1: - # Find all occurrences of this column name - indices = [i for i, x in enumerate(df.columns) if x == col] - # Rename all but the first occurrence - for i, idx in enumerate(indices[1:], 1): - df.columns.values[idx] = f"{col}_{i}" - logger.warning(f"Renamed duplicate columns in resource {resource_id}") - except Exception as e: - logger.error( - f"Failed to sanitize column names for resource {resource_id}: {str(e)}" - ) - return None - - except Exception as e: - import traceback - - logger.error( - f"Failed to load data for resource {resource_id}: {str(e)}\n{traceback.format_exc()}" - ) - return None - - # Create and index table within transaction - try: - with transaction.atomic(): - # Delete existing table if any - try: - existing_table = ResourceDataTable.objects.get(resource=resource) - try: - with connections[DATA_DB].cursor() as cursor: - cursor.execute(f'DROP TABLE IF EXISTS "{existing_table.table_name}"') - except Exception as drop_error: - logger.error( - f"Failed to drop existing table for resource {resource_id}: {str(drop_error)}" - ) - # Continue anyway as we'll try to recreate the table - - existing_table.delete() - except ResourceDataTable.DoesNotExist: - pass - except Exception as e: - logger.error( - f"Error handling existing table for resource {resource_id}: {str(e)}" - ) - # Continue with table creation - - # Create new table - data_table = create_table_for_resource(resource, df) - if not data_table: - logger.error(f"Failed to create table for resource {resource_id}") - return None - - # Update resource schema - try: - # Store existing schema descriptions to preserve them during re-indexing - existing_schemas: Dict[str, Dict[str, Optional[str]]] = {} - for schema in ResourceSchema.objects.filter( - resource=resource - ): # type: ResourceSchema - existing_schemas[schema.field_name] = { - "description": schema.description, - "format": schema.format, - } - - # Clear existing schema for this resource - ResourceSchema.objects.filter(resource=resource).delete() - - # Then create new schema entries for each column, preserving descriptions where possible - schema_success_count = 0 - schema_error_count = 0 - - for col in df.columns: - try: - # Determine format - always use the detected format from the data - sql_type = get_sql_type(str(df[col].dtype)) - - # Map SQL types to FieldTypes enum values - if sql_type == "BIGINT": - format_value = "INTEGER" - elif sql_type == "DOUBLE PRECISION": - format_value = "NUMBER" - elif sql_type == "TIMESTAMP": - format_value = "DATE" - elif sql_type == "BOOLEAN": - format_value = "BOOLEAN" - else: # TEXT or any other type - format_value = "STRING" - - # For description, preserve existing if available, otherwise auto-generate - description = f"Description of column {col}" - if col in existing_schemas: - existing_description = existing_schemas[col]["description"] - # Check for None and non-auto-generated descriptions - if existing_description is not None: - description = existing_description - logger.debug(f"Preserved custom description for column {col}") - - # Create the schema entry - ResourceSchema.objects.create( - resource=resource, - field_name=col, - format=format_value, - description=description, - ) - schema_success_count += 1 - except Exception as schema_error: - schema_error_count += 1 - logger.error( - f"Error creating schema for column {col} in resource {resource_id}: {str(schema_error)}" - ) - # Continue with other columns even if one fails - - logger.info( - f"Resource {resource_id} schema creation: {schema_success_count} columns succeeded, {schema_error_count} failed" - ) - except Exception as schema_error: - logger.error( - f"Failed to update schema for resource {resource_id}: {str(schema_error)}" - ) - # Continue and return the data_table even if schema update fails - - return data_table - except Exception as tx_error: - import traceback - - logger.error( - f"Transaction error for resource {resource_id}: {str(tx_error)}\n{traceback.format_exc()}" - ) - return None - - except Exception as e: - import traceback - - logger.error( - f"Unexpected error indexing resource {resource_id}: {str(e)}\n{traceback.format_exc()}" + # Build the complete new table; readers keep using the old one meanwhile. + build_data_table( + new_row.table_name, loaded.frame, describe_data_table(resource, built_at=timezone.now()) ) + + # Switch the resource over to the new table in one commit. + with transaction.atomic(): + # Hold the resource, so two uploads of it switch one after the other. + lock_resource(resource) + + # If a newer file was saved while we built, that upload wins: back off. + if stored_file_identity(resource) != loaded.file: + raise NewerFileSaved() + + # Retire the old row; its table is dropped once this commits. + previous_created = retire_data_table_row(resource) + + # Point the resource at the new table, keeping its place as the dataset's default. + save_data_table_row(new_row, created=previous_created) + + # Rebuild the column list, keeping the descriptions people wrote. + rebuild_resource_schema(resource, loaded.frame) + + # It only counts as switched if the new row really is there after the commit. + switched = data_table_row_exists(new_row.pk) + if not switched: + logger.error("data_index_switch_not_saved", resource_id=resource_id) + except NewerFileSaved: + logger.info("data_index_skipped_newer_file", resource_id=resource_id) + except Exception as error: + logger.error("data_index_failed", resource_id=resource_id, **error_fields(error)) + finally: + # Short of a confirmed switch, drop this build's table, unless its row was saved + # after all (e.g. a run stopped while the old table's drop was waiting). + if not switched: + drop_unswitched_table(new_row) + + if not switched: return None + logger.info("data_index_switched", resource_id=resource_id, table_name=new_row.table_name) + return new_row def query_resource_data(resource: Resource, query: str) -> Optional[pd.DataFrame]: @@ -388,29 +211,28 @@ def get_entry_count_from_file(resource: Resource) -> int: return 0 fmt = file_details.format.lower() - filepath = file_details.file.path - - if fmt in ("yml", "yaml"): - import yaml - - with open(filepath, "r") as f: - data = yaml.safe_load(f) - if isinstance(data, list): - return len(data) - if isinstance(data, dict): - for v in data.values(): - if isinstance(v, list): - return len(v) - return len(data) + if fmt not in FILE_COUNT_FORMATS: return 0 - if fmt == "xml": + with local_copy(file_details.file) as filepath: + if fmt in ("yml", "yaml"): + import yaml + + with open(filepath, "r") as f: + data = yaml.safe_load(f) + if isinstance(data, list): + return len(data) + if isinstance(data, dict): + for v in data.values(): + if isinstance(v, list): + return len(v) + return len(data) + return 0 + import xml.etree.ElementTree as ET root = ET.parse(filepath).getroot() return len(list(root)) - - return 0 except Exception as e: logger.error(f"Error counting entries from file for resource {resource.id}: {str(e)}") return 0 @@ -474,3 +296,217 @@ def get_preview_data(resource: Resource) -> Optional[PreviewData]: ) return None + +# Maximum rows that can be returned in a single fetch_resource_data call +MAX_FETCH_LIMIT = 10000 +DEFAULT_FETCH_LIMIT = 100 + + +class DataFetchError(Exception): + """Raised when fetch_resource_data receives invalid input.""" + + +def get_resource_columns(resource: Resource) -> List[str]: + """Return the list of indexed column names for a resource. + + Falls back to inspecting the data_db table if no ResourceSchema rows exist. + """ + cols = list( + ResourceSchema.objects.filter(resource=resource).values_list("field_name", flat=True) + ) + if cols: + return cols + # Fallback: introspect the table directly + try: + data_table = ResourceDataTable.objects.get(resource=resource) + with connections[DATA_DB].cursor() as cursor: + cursor.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = %s ORDER BY ordinal_position", + [data_table.table_name], + ) + return [row[0] for row in cursor.fetchall()] + except ResourceDataTable.DoesNotExist: + return [] + + +def _column_ident(column: str) -> pg_sql.Identifier: + """Quote a column name for a query that will be run with parameters. + + psycopg2 fills in ``%s`` placeholders by %-formatting the whole query + text, quoted names included, so a header like "Literacy rate (%)" breaks + the query unless its ``%`` is doubled. Every query in + ``fetch_resource_data`` is run with a parameter list, so the doubled + ``%%`` always comes back out as ``%``. + """ + return pg_sql.Identifier(column.replace("%", "%%")) + + +def _parse_filter_key(key: str) -> Tuple[str, str]: + """Split 'col__op' style filter key into (column, op). Defaults op to 'eq'.""" + if "__" in key: + col, op = key.rsplit("__", 1) + if op not in _FILTER_OPERATORS: + # No valid operator suffix — treat full key as column with eq + return key, "eq" + return col, op + return key, "eq" + + +def _build_where_clause( + filters: Dict[str, Any], allowed_columns: List[str] +) -> Tuple[pg_sql.Composable, List[Any]]: + """Build a parameterized WHERE clause from a filters dict. + + Filters are of the form ``{"column": value}`` for equality, or + ``{"column__op": value}`` for other operators. Unknown columns are rejected. + """ + if not filters: + return pg_sql.SQL(""), [] + + allowed_set = set(allowed_columns) + clauses: List[pg_sql.Composable] = [] + params: List[Any] = [] + + for raw_key, value in filters.items(): + col, op = _parse_filter_key(raw_key) + if col not in allowed_set: + raise DataFetchError(f"Unknown filter column: {col}") + op_template, transformer = _FILTER_OPERATORS[op] + + col_ident = _column_ident(col) + if op in ("isnull", "notnull"): + # Boolean toggle: isnull=true means IS NULL, isnull=false means IS NOT NULL + truthy = value not in (False, "false", "False", 0, "0", None) + sql_op = "IS NULL" if (op == "isnull") == truthy else "IS NOT NULL" + clauses.append(pg_sql.SQL("{col} {op}").format(col=col_ident, op=pg_sql.SQL(sql_op))) + continue + + # Compose: (where op_template contains %s placeholders) + clauses.append(pg_sql.SQL("{col} ").format(col=col_ident) + pg_sql.SQL(op_template)) + params.append(transformer(value) if transformer else value) + + where_sql = pg_sql.SQL(" WHERE ") + pg_sql.SQL(" AND ").join(clauses) + return where_sql, params + + +def _build_order_by(order_by: Optional[List[str]], allowed_columns: List[str]) -> pg_sql.Composable: + """Build a parameterised ORDER BY clause. Each entry may be 'col' or '-col'.""" + if not order_by: + return pg_sql.SQL("") + allowed_set = set(allowed_columns) + parts: List[pg_sql.Composable] = [] + for item in order_by: + direction = "ASC" + col = item + if item.startswith("-"): + direction = "DESC" + col = item[1:] + elif item.startswith("+"): + col = item[1:] + if col not in allowed_set: + raise DataFetchError(f"Unknown order_by column: {col}") + parts.append(pg_sql.SQL("{col} ").format(col=_column_ident(col)) + pg_sql.SQL(direction)) + return pg_sql.SQL(" ORDER BY ") + pg_sql.SQL(", ").join(parts) + + +def _build_select_list(columns: List[str]) -> pg_sql.Composable: + """Build the comma-separated list of quoted column names to SELECT.""" + return pg_sql.SQL(", ").join(_column_ident(c) for c in columns) + + +def fetch_resource_data( + resource: Resource, + filters: Optional[Dict[str, Any]] = None, + columns: Optional[List[str]] = None, + limit: int = DEFAULT_FETCH_LIMIT, + offset: int = 0, + order_by: Optional[List[str]] = None, + count: bool = True, +) -> Dict[str, Any]: + """Fetch indexed data for a Resource from data_db with column-level filtering. + + Returns a dict:: + + { + "columns": [...], # selected column names + "rows": [[...], ...], # list of rows (one list per row) + "total": , # total matching rows (None if count=False) + "limit": , + "offset": , + } + + Args: + resource: The Resource whose indexed data should be fetched. + filters: Optional dict of ``{"col": val}`` or ``{"col__op": val}`` filters. + columns: Optional list of columns to project. Defaults to all columns. + limit: Max rows to return (capped at MAX_FETCH_LIMIT). + offset: Number of rows to skip. + order_by: Optional list of columns; prefix with ``-`` for DESC. + count: When True (default) also returns the total matching row count. + + Raises: + DataFetchError: If the resource has no indexed data, or filters/columns + reference unknown columns. + """ + try: + data_table = ResourceDataTable.objects.get(resource=resource) + except ResourceDataTable.DoesNotExist: + raise DataFetchError(f"Resource {resource.id} has no indexed data table") + + allowed_columns = get_resource_columns(resource) + if not allowed_columns: + raise DataFetchError(f"Resource {resource.id} has no schema/columns available") + + # Validate and resolve projected columns + if columns: + unknown = [c for c in columns if c not in allowed_columns] + if unknown: + raise DataFetchError(f"Unknown columns: {unknown}") + select_columns = columns + else: + select_columns = allowed_columns + + # Clamp pagination + if limit is None or limit <= 0: + limit = DEFAULT_FETCH_LIMIT + limit = min(int(limit), MAX_FETCH_LIMIT) + offset = max(int(offset or 0), 0) + + table_ident = pg_sql.Identifier(data_table.table_name) + cols_sql = _build_select_list(select_columns) + where_sql, params = _build_where_clause(filters or {}, allowed_columns) + order_sql = _build_order_by(order_by, allowed_columns) + + select_query = ( + pg_sql.SQL("SELECT ") + + cols_sql + + pg_sql.SQL(" FROM ") + + table_ident + + where_sql + + order_sql + + pg_sql.SQL(" LIMIT %s OFFSET %s") + ) + + total: Optional[int] = None + with connections[DATA_DB].cursor() as cursor: + # Safety: cap query time + cursor.execute("SET statement_timeout = 10000") + + if count: + count_query = pg_sql.SQL("SELECT COUNT(*) FROM ") + table_ident + where_sql + cursor.execute(count_query, params) + row = cursor.fetchone() + total = int(row[0]) if row else 0 + + cursor.execute(select_query, params + [limit, offset]) + result_columns = [desc[0] for desc in cursor.description] + rows = [list(r) for r in cursor.fetchall()] + + return { + "columns": result_columns, + "rows": rows, + "total": total, + "limit": limit, + "offset": offset, + } diff --git a/api/utils/data_indexing_architecture.md b/api/utils/data_indexing_architecture.md new file mode 100644 index 00000000..adbc6388 --- /dev/null +++ b/api/utils/data_indexing_architecture.md @@ -0,0 +1,342 @@ +# Data index — per-upload data tables + +> Backend only. Related: [`docs/dataset_data_api.md`](../../docs/dataset_data_api.md) (the public data API that reads these tables). Plan: `plans/claude/data-index-table-names.md`. + +## Overview + +Every tabular resource file (csv, xls, xlsx, ods, parquet, feather, json, tsv) is copied into a table of its own in the separate `data_db` Postgres database. Previews, row counts, the column list, the data API (`/api/resources//data/`, `/api/datasets//data/`) and chart queries read those tables. A resource's `ResourceDataTable` row (main database) records which table is its own; every reader looks the table up from that row, never from a name rule. + +This feature makes each upload's table **unique and safely replaced**: + +- **Names come from the upload's own row id:** `rdt_<32 hex of the ResourceDataTable id>`, 36 characters. They never collide and never change when a resource or dataset is renamed. +- **Re-uploads build first, then switch, then drop.** The new table is built completely, one main-database commit points the resource at it, and the old table is dropped only after that commit. Readers see the complete old data, then the complete new data. A failed re-upload leaves the old data exactly as it was. +- **Deletes free storage.** Deleting a resource, a dataset (by cascade), or a row in the admin drops the table once the delete commits. A rolled-back delete keeps it. +- **Existing tables are migrated by a one-off command** (`migrate_data_table_names`). Its runbook is below. + +Why it was needed: names used to be `slug(resource name)[:50]_slug(dataset title)[:50]_<8 hex>`, up to 110 characters. Postgres keeps only the first 63 bytes of any name, so the id suffix was usually cut off and two resources with long, similar names shared one physical table. A re-upload of one replaced the other's data, and the other's preview, charts and data API served the wrong numbers. Re-uploads also dropped the old table before building the new one: readers saw empty or half-written data for a few seconds, a failed re-upload lost the data for good, and deletes never dropped anything. + +## Submodule map + +| Submodule | Trigger | Code | +|---|---|---| +| Upload / re-index flow | GraphQL `createFileResources`, `updateFileResource` (with a file), `resetFileResourceSchema`, `updateFileResourceSchema` (when a column format changes); `python manage.py update_dataindex` | `index_resource_data` in `api/utils/data_indexing.py`; helpers in `api/utils/data_tables.py` | +| Table names | used by the model and the flow | `api/utils/data_table_names.py` | +| Delete clean-up | `post_delete` of any `ResourceDataTable` row (re-upload, resource/dataset delete, admin delete) | `api/signals/data_table_signals.py` → `drop_data_table_if_unshared` | +| Readers | GraphQL resource fields (preview, row count), data API views, chart generator | `get_row_count`, `get_preview_data`, `get_resource_columns`, `fetch_resource_data`, `query_resource_data` in `api/utils/data_indexing.py` (unchanged) | +| Migration of existing tables | `python manage.py migrate_data_table_names` (one-off, operator-run) | `api/management/commands/migrate_data_table_names.py`; helpers in `api/utils/data_table_migration.py` | + +--- + +## Submodule: Upload / re-index flow + +### Trigger +`index_resource_data(resource, load_timeout=60)` is called synchronously by the resource file mutations in `api/schema/resource_schema.py` (none of them runs inside a transaction), by `update_dataindex` per resource, and by the migration command for rebuilds (with `load_timeout=None`). + +### Business use case +A publisher uploads or replaces a tabular file. Visitors then preview it, chart it and query it through the data API. They must never see another resource's data, an empty table mid-replace, or lose the old data because a new file was bad. + +### Flow (English) +1. **Refuse inside an outer transaction.** If the main database is already in an atomic block, log `data_index_refused_inside_transaction` and return None. Inside someone else's transaction, "committed" would only mean a savepoint was released, and a later rollback would leave a switched table pointing nowhere. +2. **Check the file.** The resource must have a stored file in an indexed format; otherwise return None. Nothing is built. +3. **Load the file** into a frame, and note which stored file it was (storage name + `sha256`). Loading is limited to 60 s where the platform allows it: SIGALRM works only in a process's main thread, so management commands get the limit and web worker threads never have. Column names are cleaned: double quotes removed, line breaks and tabs become spaces, and names that then collide get `_1`, `_2`… An unreadable file, a timeout or an empty frame returns None. +4. **Make this upload's row in memory.** A new `ResourceDataTable` with its UUID, so its table name `rdt_` is known before anything is saved. +5. **Build the new table** in one `data_db` transaction: create it, copy the rows in through a temporary table (`ON COMMIT DROP`), and store a JSON comment (resource id and name, dataset title, `built_at`). A failure, such as a value out of range mid-copy, rolls the whole table back, and the old row and table are untouched. +6. **Switch, in one main-database transaction:** + 1. lock the resource row (`select_for_update`), so two uploads of one resource switch one after the other; + 2. re-read the stored file's identity, and if a newer file was saved while this one was building, back off (`NewerFileSaved`), because the newest file wins; + 3. delete the old `ResourceDataTable` row, if any. Its delete receiver schedules the old table's drop for after the commit; + 4. save the new row, carrying over the old row's `created` (see "Dataset default table" below); + 5. rebuild the `ResourceSchema` column rows (descriptions people wrote are kept by column name). Errors escape, and a header longer than the 255-character `field_name` is refused up front. +7. **Confirm the switch.** It counts only if the new row exists after the transaction. +8. **`finally`: drop this build's table unless step 7 confirmed the switch.** That covers a failed build, a failed switch, a newer file, a missing row, and even `KeyboardInterrupt`. Before dropping, it checks the database: if the new row was saved after all (a command stopped with Ctrl+C while the old table's after-commit drop waited for a reader, or a commit that went through but reported an error), the table is the resource's live data and stays. If the check itself fails, the table is left for the orphan audit. +9. After the commit, the receiver drops the old table (see Delete clean-up). + +Returns the new `ResourceDataTable`, or None when the resource was not switched (each reason logged). + +**Why the switch lets errors escape.** The old column rebuild caught each column's error. Inside an atomic block a database error marks the whole block for rollback, so one bad header silently rolled the switch back while the upload still looked successful. + +**Why the newest file wins.** Each upload request saves its own file before indexing it. Without the re-check, a slow build of an older file could switch in last: the download would serve one file while previews, charts and the data API served another. + +**Dataset default table.** `GET /api/datasets//data/` without `resource_id` serves the dataset's data table with the oldest `created`. The new row inherits the old row's `created`, so replacing a file never moves a dataset's default to another resource. The migration doesn't reshuffle defaults either. + +### Helpers (`api/utils/data_tables.py`, Tier 2 unless noted) +- `is_inside_transaction() -> bool`: whether the main DB is in an atomic block. +- `has_indexable_file(resource) -> bool`: has a stored file whose format is in `INDEXED_FORMATS`; logs why not. +- `load_resource_frame(resource, timeout=60) -> LoadedFile | None`: frame plus `FileIdentity(name, sha256)`; None (logged) when unreadable, too slow, or empty. Depends on Tier 1 `local_copy`, `load_tabular_data` (`api/utils/file_utils.py`). +- `stored_file_identity(resource) -> FileIdentity | None`: the current file, read fresh from the database. +- `new_data_table_row(resource) -> ResourceDataTable`: unsaved, named `rdt_`. +- `describe_data_table(resource, built_at, unverified=False) -> str`: the table comment (JSON). `unverified` is used only by the migration. +- `build_data_table(table_name, frame, comment)`: one `data_db` transaction. Raises on failure, leaving nothing behind. Names via `sql.Identifier`, comment via `sql.Literal`. +- `lock_resource(resource)`: `select_for_update` on the resource row (no-op on SQLite). +- `retire_data_table_row(resource) -> datetime | None`: deletes the current row, returns its `created`. +- `save_data_table_row(row, created)`: inserts, then sets `created` with a queryset update (it's `auto_now_add`). +- `rebuild_resource_schema(resource, frame)`: replaces column rows. Raises `ValueError("column N header is longer than 255 characters")` and lets DB errors escape. +- `data_table_row_exists(row_id) -> bool`. +- `drop_unswitched_table(row)`: the flow's clean-up. It drops the build's table only if its row isn't saved, and leaves it when the check fails. Never raises. +- `drop_data_table(name) -> bool`: `DROP TABLE IF EXISTS` with `SET LOCAL lock_timeout = '5s'`. Never raises: a failure is logged (`data_table_drop_failed`) and the table is left as an orphan for the migration command to report. +- `error_fields(error) -> dict`: `error_type`, Postgres `error_code`, and `where` (innermost frame in `api/`). **Never the message**, because COPY errors quote the offending cell value. +- `get_sql_type(dtype) -> str`: BIGINT / DOUBLE PRECISION / TIMESTAMP / BOOLEAN / TEXT (moved here; still importable from `data_indexing`, as are `DATA_DB` and `INDEXED_FORMATS`). + +`api/utils/data_table_names.py` (no model imports, because the model uses it): +- `data_table_name(row_id) -> "rdt_" + hex`. +- `physical_table_name(name) -> str`: the first 63 bytes, never half a character (what Postgres stores). +- `may_share_physical_table(name) -> bool`: only names of ≥ 60 bytes can be the cut of a longer one. + +### Model rule +`ResourceDataTable.save()` fills a blank `table_name` with `data_table_name(self.id)`. Existing rows keep their stored names. Nothing else generates names. Rolling the code back is safe: old code only generates a name when the field is blank, and every reader uses the stored name. + +--- + +## Submodule: Delete clean-up + +### Trigger +`post_delete` with `sender=ResourceDataTable`. Django sends it for every row in a cascade or a queryset delete, because a receiver is connected (no fast delete). That covers re-upload (`retire_data_table_row`), `deleteFileResource`, `deleteDataset`, admin deletes of a row, resource or dataset, and the migration's rebuilds. + +### Flow (English) +1. The receiver schedules `drop_data_table_if_unshared(table_name, row_id)` with `transaction.on_commit(..., using=, robust=True)`. +2. After the commit, the drop helper checks whether another row still maps to the same physical table. This is possible only for old long names; `rdt_` names skip the check. + 1. If one does, it logs `data_table_drop_skipped_shared` and keeps the table. + 2. Otherwise it drops the table (5 s lock timeout). Any failure is logged and swallowed. + +A rolled-back delete never runs the callback, so its data is kept. + +**Registration.** The module is imported by `api/signals/__init__.py`. A test checks the receiver is connected in a fresh interpreter, because `collaborative_signals.py` once shipped without being imported and never ran. + +**Lock order.** A switch locks the resource, then its data-table row. A resource delete locks the rows, then the resource. A switch and a delete of the same resource at the same moment can therefore deadlock. Postgres aborts one of them, and nothing is lost: the aborted switch drops its own table, and the aborted delete leaves everything in place. + +--- + +## Submodule: Readers (unchanged) +`get_row_count`, `get_preview_data`, `get_resource_columns` (falls back to `information_schema` by table name when there are no column rows), `fetch_resource_data` (data API; column names via `_column_ident`, which doubles `%`), and `query_resource_data` (charts; `{{table}}` replaced by the quoted table name). They all look up `ResourceDataTable` per call and work unchanged on `rdt_` tables. A reader that fetched the old row just before a switch can hit "relation does not exist" once the old table is dropped. It then takes its existing error path (None / 0 / the data API's 500); it never gets the new file's rows under the old row. Readers still compose the table name with f-strings; `rdt_` names are safe in that position. + +--- + +## Submodule: Migration of existing tables — `migrate_data_table_names` + +### Trigger +Operator-run, once after deploy (and safe to rerun any time as an orphan audit). + +### Business use case +Move every existing table to an `rdt_` name and repair tables two resources were sharing. Then list, and after review drop, tables nothing uses (left behind by deletes from before this feature). + +### Flow (English) +1. Check the options. `--drop-orphans` needs `--apply` and `--from-report`, and `--report` must not name the reviewed file. +2. **Safety:** refuse if `data_db` and the main database name the same host, port and database, or if `data_db` holds `django_migrations` or any Django model's table. Either would make every application table look like an orphan. +3. Read the reviewed orphan list now, if dropping. A missing or malformed file stops the run before any change. +4. Write an "in progress" report. This proves the path is writable, and a run that dies leaves a trace. +5. Read every row (oldest first) and every `data_db` table (size and comment), and sort each row (limited to `--resource-id` if given): + - **already migrated**: `rdt_` exists; + - **row update**: a crashed earlier run renamed the table but didn't update the row (the old name is gone, `rdt_` exists); + - **rename**: an old name of ≤ 63 bytes. It can't have collided; + - **rebuild**: an old name of > 63 bytes, whether one row or several point at the physical table. It may already hold another resource's data, so it is never just renamed while its file is usable. Each rebuild carries whether the table is shared; + - **missing table**: neither name exists. +6. **Dry run (the default):** change nothing, but check each rebuild's and missing table's stored file, and record the planned outcome: + - `planned_rebuild`; + - `planned_unverified_rename`: the file is missing or unusable and the table is not shared; + - `planned_missing_file` / `planned_not_indexable`: the table is shared, so it is left for a person; + - `planned_reindex`. +7. **`--apply`**, in this order: + 1. **Row updates.** + 2. **Renames:** in a main-DB transaction, lock the resource and re-check that the row still has its old name (otherwise `changed`). Then, in `data_db`, `SET LOCAL lock_timeout = '2s'`, rename the table and set its comment. Update the row last. A lock timeout gives `skipped_lock`, and a rerun finishes it. + 3. **Rebuilds:** oldest first, through `index_resource_data(resource, load_timeout=None)`. The old row's delete receiver keeps a shared table until its last row has moved. + - If the file is missing or can no longer become a table (for example, later replaced by a PDF) and no other row uses the table, it is renamed and **marked `"unverified": true` in its comment** (`unverified`). + - If the table is shared, it is left (`missing_file` / `not_indexable`). Once its sharers have moved, the next run finds it alone and marks it unverified. + 4. **Missing tables:** re-indexed from the file, or `missing_file` / `not_indexable`. +8. **`--drop-orphans`:** recompute orphans now, and drop only those that are both in the reviewed report and still orphans. +9. Write the full report and print the summary. Every row's outcome is also logged as it happens (`data_table_migration_row`). "nothing to do" is printed when no row needs a rerun: every row is migrated, or needs a person (missing or unusable file). + +**Orphans:** +- **Never orphans:** a Django model's table, any row's physical table or `rdt_`, and an `rdt_` table whose comment says it was built under an hour ago (its upload may still be switching). +- **Listed:** `rdt_` tables without a readable build time, because every table the new code builds has a comment written in the same transaction as its creation. +- Each listed orphan carries its size and comment. + +**Unverified tables** are listed in every report (`unverified_tables`), dry run or apply, until the resource is re-uploaded (the new table has no mark). They may hold another resource's data. Restore the resource's file and re-upload it, or check the data by hand. + +**Report** (JSON, default `data_table_migration_.json` in the working directory): +- `mode`, `status`; +- `counts` by outcome and `shared_groups`; +- `rows` (ids, table name, action, outcome, `shared`); +- `needs_manual_fix`; +- `unverified_tables`; +- `orphans` (name, size, comment), `orphans_dropped`; +- `access_models_touched`: access models whose column choices are reset because the resource's column rows are recreated. This is existing re-upload behaviour. + +No file contents or values appear in the report. + +### Runbook +1. Deploy. New uploads use `rdt_` names from now on, and old names keep working. +2. Off-peak: `python manage.py migrate_data_table_names --report dry.json`. Review `counts`, `shared_groups`, `needs_manual_fix` (restore files first if you can), `unverified_tables` and `orphans`. +3. `python manage.py migrate_data_table_names --apply --report apply1.json`. Rerun until it prints "nothing to do". + - `skipped_lock` and `failed` rows are retried. + - A row that comes back `failed` on two runs in a row needs a person: its file is stored and in a supported format but won't load or switch (it doesn't parse, has only a header, or has a header over 255 characters). Find it under `rows` with outcome `failed`, using the resource id and the `data_index_failed` / `data_index_load_failed` log lines. Its old table keeps serving unmarked, and a long one may hold another resource's data. + - A shared table whose file is missing is marked on the run after its sharers move. +4. At least an hour later: `python manage.py migrate_data_table_names --report orphans.json`. Review the orphan list and remove from the file any table you want kept. +5. `python manage.py migrate_data_table_names --apply --drop-orphans --from-report orphans.json --report drop.json`. +6. Follow up on `unverified_tables` and `needs_manual_fix` by hand. + +Renames take an exclusive lock briefly and wait up to 2 s behind running reads. Rebuilds hold two copies of a table for a moment. + +**Old naming schemes the command meets:** +- `resource_data_<32 hex>`: March–May 2025, 46 characters, so renamed. +- Raw `--<32 hex>`: May–July 2025. These can be mixed case, contain spaces or non-ASCII, and are usually over 63 bytes, so they are rebuilt. +- `slug(name)[:50]_slug(title)[:50]_<8 hex>`: from July 2025. Short ones are renamed and long ones rebuilt. + +Short names of every scheme are unique, so they can't have collided. + +### Helpers (`api/utils/data_table_migration.py`, plus `list_data_db_tables`, `django_table_names`, `assert_data_db_is_separate`, `table_built_at`, `table_is_unverified` in `data_tables.py`) +- `read_data_table_rows()`: every row, oldest first, as `DataTableRow(row_id, resource_id, table_name, created)`. +- `plan_table_migration(rows, tables, now, resource_id=None) -> MigrationPlan`: pure; the sorting above, plus `orphans` and `shared_groups`. +- `find_orphans(rows, tables, now)`: pure; the orphan rules. +- `file_problem(resource)`: `missing_file`, `not_indexable` or None. Storage errors are raised. +- `rebuild_decision(row, shared)` / `reindex_decision(row)`: what `--apply` will do (`check_failed` if the file check errors). +- `point_row_at_its_table`, `rename_legacy_table(row, unverified=False)`, `rebuild_from_file(row, shared)`, `reindex_missing_table(row)`: the actions. Each returns an outcome string and never raises. +- `find_unverified(rows, tables)`: rows whose table carries the mark. +- `access_models_touched(resource_ids)`, `read_reviewed_orphans(path)` (CommandError when unreadable), `same_file`, `drop_reviewed_orphans(reviewed, current)`, `record_outcome` (logs and returns the report line), `nothing_left_to_do`, `build_report`, `write_report`, `summary_lines`. + +--- + +## Data model +- `ResourceDataTable` (`resource_data_table`): `id` UUID pk, `resource` one-to-one, `table_name` CharField(255) unique, `created`, `modified`. **No schema change and no migration file.** Only the generated value changed. `created` is now carried over on re-upload. +- `ResourceSchema`: rebuilt on every re-index (existing behaviour). `AccessModelResource.fields` (many-to-many to these rows) loses its selections whenever they are rebuilt, as before. +- `ResourceFileDetails.file` and `sha256` are read as the file identity (`sha256` is computed on save when the file changes). +- `data_db` tables: one per tabular resource, named `rdt_`, with a JSON comment: `resource_id`, `resource`, `dataset`, `built_at`, and `unverified` when the migration kept a table without a rebuild. +- No Elasticsearch documents are affected. + +## Indexing & performance +- **Switch:** `Resource` by pk (`select_for_update`); `ResourceDataTable` by `resource_id` (one-to-one unique index); `ResourceFileDetails` by `resource_id` (unique). +- **Receiver guard:** `ResourceDataTable.table_name LIKE '<63-byte prefix>%'`, only for old names of ≥ 60 bytes (`rdt_` names skip it). This uses the unique btree on `table_name`, but may be a sequential scan on non-C collations. That's acceptable at one row per tabular resource (thousands). +- **Command:** one ordered read of `ResourceDataTable` (values only, iterator) and one `pg_class` catalogue query on `data_db`. Each is repeated once at the end, and once more when dropping orphans. +- **`data_db`:** no indexes added. Readers do full-table reads with existing timeouts. +- **Disk:** a re-upload briefly holds two copies of the table. +- No new user-facing query, so no pagination change. + +## Security +- No new endpoint and no permission change. The file mutations keep their `ChangeResourcePermission` (edit rights on the dataset) and the data API keeps its existing check. If a user from another organization sends the same mutation, it is refused exactly as before (`tests/schema/test_resource_permissions.py`). +- Out of scope, and unchanged: the public row API serves rows of any PUBLISHED dataset regardless of `access_type`. +- All new SQL composes table names with `sql.Identifier` and comments with `sql.Literal`. Resource and dataset names reach SQL only as a literal comment. +- The orphan sweep refuses to run against a database that looks like the main one, never drops Django tables, and drops only names a person reviewed that are still unused. +- Logs carry ids, outcomes, counts and error codes. Never file contents, cell values or database error messages. + +## SDK impact +SDK: no. This is storage-internal, and no public API changes shape. + +## Tests + +Tests marked `data_db` run against a real Postgres. Locally: + +``` +docker run -d --name dataspace-test-data-db -e POSTGRES_PASSWORD=postgres -p 55432:5432 postgres:16 +export TEST_DATA_DB_URL=postgres://postgres:postgres@localhost:55432/postgres +pytest tests/data_index +``` + +How the setup works: +- **Database:** `tests/test_settings.py` adds the `data_db` alias only when `TEST_DATA_DB_URL` is set. pytest-django creates and drops `test_dataspace_data_db`, and `tests.data_index.routers.DataDbTestRouter` keeps Django models out of it. +- **Skipping:** without the URL these tests are skipped, unless `REQUIRE_DATA_DB_TESTS=1` (set in CI, which runs a `postgres:16` service). Then the run fails, because a run that skipped them proves nothing. +- **Clean-up:** an autouse fixture drops every `public` table after each such test, since pytest-django only empties tables of models routed there, and there are none. +- **Where it applies:** the skip/fail hook and that fixture live in `tests/data_index/conftest.py`, and `uses_data_db` in `tests/data_index/support.py`. Tests elsewhere that need them must move both into a shared conftest. +- **Main database:** stays in-memory SQLite, so row locks are no-ops there. +- **Locally:** if `drf_yasg` fails with "No module named pkg_resources", install `setuptools<81` (CI installs it). + +### Layer 1 — DB helper tests +- **Naming:** a new row is named `rdt_`, and an existing row keeps its stored name on save. Two resources with names identical for 60+ characters get their own tables with their own rows. +- **Uploads:** a first upload creates the row, the table, the column rows and the comment. A re-upload switches to a new table, drops the old one and keeps custom descriptions. +- **Switch timing:** at the moment of the switch the new table is full and the old one still exists, and the lock is taken inside the switch transaction. The old table is dropped only after commit, with no transaction open and the new row saved. +- **Failures keep the old data:** a value out of range mid-copy keeps the old row and table, leaves no new table, and logs error code 22003 without the value. A 300-character header fails the switch, keeps the old row, table and column rows, and drops the new table. +- **Stopped runs:** a run stopped (KeyboardInterrupt) while the old table's after-commit drop waits keeps the new table the resource already points at. The clean-up keeps a table whose row is saved, drops one whose row isn't, and leaves it when the check fails. +- **Readers and races:** a reader still holding the old name gets "does not exist", never new rows. A slower build of an older file backs off when a newer file was saved, and the newer one stays. +- **Transaction and default:** indexing inside an outer transaction is refused and builds nothing. A re-upload keeps the row's `created` and the dataset's default table in the data API. +- **Deletes:** deleting a resource drops its table; deleting a dataset drops all its tables; a rolled-back delete keeps the table. Two legacy rows sharing a table: the table is kept until the last one is deleted. A drop of a table that's already gone doesn't fail. A drop blocked past the lock timeout is logged and the delete still succeeds. +- **Readers (regression):** on an `rdt_` table with a `%` header, the row count, preview, column list (including the `information_schema` fallback), filtered and ordered fetch, and a chart query all work. +- **Helpers:** indexable-file checks; load records the file identity, returns None for a header-only file, and cleans headers; the column rebuild lets DB errors escape and refuses over-long headers. +- **Migration — moving tables:** + - a short table is renamed only on `--apply`; + - a shared long table is rebuilt from each resource's own file, and the shared table is dropped after both rows move; + - a single long table is rebuilt, not renamed; + - with its file gone, a single long table is renamed and marked unverified, and later runs still list it; + - with its file gone, a shared row stays while the other row moves; + - a run after a crash just updates the row; + - a rename blocked by a reader is skipped, then finished on a rerun. +- **Migration — orphans:** listed with size; drop needs `--from-report`; only reviewed names that are still orphans are dropped; young `rdt_` tables are never listed. +- **Migration — reruns and reports:** + - a second `--apply` has nothing to do; + - access models are listed; + - the dry run shows the planned outcome per long table, the shared flag and `shared_groups`; + - an unusable file (PDF) is marked unverified, or left while shared, and the runs settle on "nothing to do"; + - rebuilds read with no time limit; + - a missing table is re-indexed or left for a person; + - `--resource-id` limits the run; + - a rename backs off when an upload replaced the row after planning. +- **Migration — refusals:** a bad reviewed report stops the run before any change; the report may not overwrite the reviewed file; the run refuses when the data database is the main one or holds `django_migrations`. + +### Layer 2 — Non-DB helper tests +- **Names:** `rdt_` plus lowercase hex, 36 characters. The physical name is cut at 63 bytes and never mid-character. +- **Comment:** a resource name with quotes and semicolons is stored as text, not run as SQL. +- **Orphan rules:** Django tables, a row's tables, and fresh builds are excluded. +- **Registration:** the receiver is connected after app load (fresh interpreter). + +### Layer 3 — Flow tests +- **Upload flow order:** load → name → build → lock → file check → retire → save → rebuild columns → confirm. +- **Clean-up:** the unswitched-build clean-up runs when the build, lock, retire or column rebuild fails, on a newer file, when the row is missing after commit, and on `KeyboardInterrupt`. It never runs after a confirmed switch. +- **Nothing to index:** nothing is built when inside a transaction, when there is no indexable file, or when nothing loaded. +- **Receiver:** schedules one guarded drop per deleted row, via `on_commit(robust=True)`. +- **Command:** the safety check runs first; a dry run changes nothing and writes the report; `--apply` runs row updates → renames → rebuilds (in plan order) → re-indexes; a failed safety check stops before any read; orphans are dropped only with `--apply` and a reviewed report. + +### Layer 4 — Backend API e2e +As the dataset owner, through the GraphQL schema and the data API: +- `createFileResources` gives an `rdt_` table, and the data API returns its rows; +- `updateFileResource` with a new file serves the new rows and the old table is gone; +- `resetFileResourceSchema` switches to a fresh table; +- `deleteFileResource` drops the table, and the data API returns 404; +- `deleteDataset` drops every table. + +Cross-org denial is unchanged and covered by `tests/schema/test_resource_permissions.py`. + +### Layer 5 — API user journey +`tests/journeys/data_index/reupload-and-delete.py`, on request against a running backend (`KEYCLOAK_TEST_TOKEN`, `TEST_DATASET_ID`, `TEST_BASE_URL`): +1. Upload a CSV and read its rows. +2. Replace it while a thread polls the data API. Every read is the complete old file or the complete new one. +3. Fire two replacements at once. The data API serves the same file the download serves. +4. Delete the resource. The data API returns 404. + +At least an hour later, the operator's dry run should list no orphan from the racing uploads. + +### Layer 6 — Browser e2e +n/a — no UI change. + +### Visual checklist (manual, on request) +n/a — no UI change. + +### LLM-judge points +n/a — all assertions are deterministic. + +--- + +## Failure model + +| What could go wrong | Guarantee | +|---|---| +| Two resources map to one physical table | `rdt_` names are unique by construction. | +| A reader sees an empty or half-filled table | The new table is built fully before the switch, and the old one is dropped only after commit. | +| A failed re-upload wipes the data | The old row and table are untouched until the switch commits. | +| A failed build or switch leaves a stray table | `finally` drops it unless the switch is confirmed. | +| A rolled-back delete loses data | Drops run only `on_commit`. | +| The receiver drops a table another row still uses | Shared-name guard for old long names. | +| The receiver isn't loaded | Imported in `api/signals/__init__.py`, and a fresh-interpreter test checks it. | +| Two re-uploads race | Resource row lock plus the newest-file check. | +| A drop fails (lock, missing table) | Logged, never raised; the table becomes an orphan the command reports. | +| A rename gets stuck behind long reads | 2 s lock timeout; skipped, reported, and finished by a rerun. | +| The command crashes between rename and row update | The rerun updates the row; the "unverified" mark survives in the comment. | +| A shared table is rebuilt from the wrong file | Each row is rebuilt from its own file, and the table is dropped only after the last row moves. | +| A stored file is missing or unusable | Reported per row. An unshared table is marked unverified and listed in every report; a shared one is left for a person. | +| A collided table whose sharer was deleted gets renamed | Long old names are always rebuilt while the file is usable. | +| The orphan sweep deletes application tables or an in-flight upload's table | Same-database and Django-table refusals, the one-hour age guard, and only reviewed names dropped. | +| A swallowed error rolls the switch back silently | Column-row errors escape, the switch is confirmed by the row existing, and indexing refuses to run inside an outer transaction. | +| Test data tables leak between tests | An autouse fixture drops every `public` table after each `data_db` test. | + +## Limitations & future work +- **TODO — a failed re-upload still reports success to the uploader.** This is existing behaviour: the upload mutations ignore the indexing result. The file is saved (the download serves it), but if its rows can't be loaded, previews, charts and the data API keep serving the previous file, until a successful re-upload. Nobody is told the two disagree. Direction (pending a talk with the product team, with the DataSpace team): check the file at upload against the database's limits and enforce them on the platform (for example, column names within the 255 characters a column row allows), and flag problem columns where people see column names and add column descriptions. A `TODO` comment marks the spot in `_create_file_resource_schema` (`api/schema/resource_schema.py`). +- **TODO — two long headers that differ only after 63 bytes** become one column in Postgres, so the build fails and that file never gets a table (the upload still reports success; see the first item). Devanagari headers hit this at about 21 characters. This existed before and is not fixed yet. A fix would catch names that only clash once cut to 63 bytes during column clean-up, and either shorten them uniquely (keeping the stored column names in step) or refuse the file with a clear reason. A `TODO` comment marks the spot in `_clean_column_names` (`data_tables.py`). +- **Access-model column selections are reset** whenever column rows are rebuilt (every re-upload, and migration rebuilds). Carrying them over by column name would be a small follow-up. +- **Readers still use f-string table names.** That is safe for `rdt_` names; they were left as they are deliberately. +- The plan's failure table said 5 s for the rename lock timeout. The design and the code use 2 s for renames and 5 s for drops. diff --git a/api/utils/data_table_migration.py b/api/utils/data_table_migration.py new file mode 100644 index 00000000..0000e231 --- /dev/null +++ b/api/utils/data_table_migration.py @@ -0,0 +1,475 @@ +"""Helpers for ``migrate_data_table_names``: moving old data tables to per-upload names. + +Old tables were named from the resource name and dataset title. Postgres cut +those names to 63 bytes, so two resources could end up sharing one table. The +command works out what each ResourceDataTable row needs, then: + + - a short old name (63 bytes or less) can't have collided, so its table is + renamed in place to "rdt_"; + - a long old name may already hold another resource's data, so the + resource is rebuilt from its own stored file (through the normal upload + flow, with no limit on reading time); the shared table is dropped by the + delete receiver once its last row has moved. If the file is gone or can't + become a table, a table only this row uses is renamed and marked + "unverified" in its comment (listed on every run until the resource is + re-uploaded); a shared one is left for a manual fix; + - a row whose table was renamed but not yet pointed at (a crashed run) just + gets its row updated; + - a row with no table at all is re-indexed from its file; + - tables no row uses are listed as orphans, and dropped only from a report + someone reviewed. + +A dry run checks each stored file too, so its report shows what --apply will +do. Each step is safe to rerun; every row's outcome is logged as it happens. +Logs carry ids, outcomes and counts only. +Docs: ./data_indexing_architecture.md +""" + +import json +import os +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple + +import structlog +from django.core.management.base import CommandError +from django.db import connections, transaction +from django.db.utils import OperationalError +from psycopg2 import sql as pg_sql # type: ignore[import-untyped] + +from api.models import AccessModelResource, Resource, ResourceDataTable +from api.utils.data_indexing import index_resource_data +from api.utils.data_table_names import ( + DATA_TABLE_PREFIX, + MAX_IDENTIFIER_BYTES, + data_table_name, + physical_table_name, +) +from api.utils.data_tables import ( + DATA_DB, + DataDbTable, + describe_data_table, + django_table_names, + drop_data_table, + error_fields, + has_indexable_file, + lock_resource, + table_built_at, + table_is_unverified, +) + +logger = structlog.get_logger("dataspace.data_table_migration") + +# How long a rename waits for readers holding the table before it is skipped. +RENAME_LOCK_TIMEOUT = "2s" +# Per-upload tables younger than this may belong to an upload that hasn't switched yet. +YOUNG_TABLE_AGE = timedelta(hours=1) +LOCK_NOT_AVAILABLE = "55P03" +# Outcomes that leave nothing for a rerun to do (missing or unusable files need a person). +NEEDS_MANUAL_FIX = { + "missing_file", + "not_indexable", + "planned_missing_file", + "planned_not_indexable", +} +SETTLED_OUTCOMES = NEEDS_MANUAL_FIX | {"already_migrated"} + + +class DataTableRow(NamedTuple): + """The fields of a ResourceDataTable row the migration looks at.""" + + row_id: Any + resource_id: Any + table_name: str + created: datetime + + +@dataclass +class MigrationPlan: + """What each row needs. Rebuilds carry whether another row shares the table.""" + + already_migrated: List[DataTableRow] = field(default_factory=list) + row_updates: List[DataTableRow] = field(default_factory=list) + renames: List[DataTableRow] = field(default_factory=list) + rebuilds: List[Tuple[DataTableRow, bool]] = field(default_factory=list) + missing_tables: List[DataTableRow] = field(default_factory=list) + orphans: List[DataDbTable] = field(default_factory=list) + shared_groups: int = 0 # old tables two or more rows point at + + +def read_data_table_rows() -> List[DataTableRow]: + """Every ResourceDataTable row, oldest first (one row per tabular resource).""" + rows = ( + ResourceDataTable.objects.order_by("created") + .values_list("id", "resource_id", "table_name", "created") + .iterator() + ) + return [DataTableRow(*row) for row in rows] + + +def plan_table_migration( + rows: List[DataTableRow], + tables: List[DataDbTable], + now: datetime, + resource_id: Optional[str] = None, +) -> MigrationPlan: + """Sort rows by what they need, given the tables that exist now. + + ``resource_id`` limits the actions to one resource; orphans are always + worked out against every row. + """ + existing = {table.name for table in tables} + rows_per_table = Counter(physical_table_name(row.table_name) for row in rows) + plan = MigrationPlan( + orphans=find_orphans(rows, tables, now), + shared_groups=sum(1 for count in rows_per_table.values() if count > 1), + ) + for row in rows: + if resource_id and str(row.resource_id) != str(resource_id): + continue + own_name = data_table_name(row.row_id) + physical = physical_table_name(row.table_name) + if row.table_name == own_name: + target = plan.already_migrated if own_name in existing else plan.missing_tables + target.append(row) + elif physical in existing and _is_long(row.table_name): + plan.rebuilds.append((row, rows_per_table[physical] > 1)) + elif physical in existing: + plan.renames.append(row) + elif own_name in existing: + plan.row_updates.append(row) + else: + plan.missing_tables.append(row) + return plan + + +def _is_long(name: str) -> bool: + return len(name.encode("utf-8")) > MAX_IDENTIFIER_BYTES + + +def find_orphans( + rows: List[DataTableRow], tables: List[DataDbTable], now: datetime +) -> List[DataDbTable]: + """Tables no row uses, in the order given. + + Never a Django model's table, never a row's (cut) table or its renamed + "rdt_" table, and never a per-upload table built less than + YOUNG_TABLE_AGE ago (its upload may still be switching over). + """ + in_use = {physical_table_name(row.table_name) for row in rows} + in_use |= {data_table_name(row.row_id) for row in rows} + protected = in_use | django_table_names() + orphans = [] + for table in tables: + if table.name in protected: + continue + built_at = table_built_at(table.comment) + if ( + table.name.startswith(DATA_TABLE_PREFIX) + and built_at + and now - built_at < YOUNG_TABLE_AGE + ): + continue + orphans.append(table) + return orphans + + +def point_row_at_its_table(row: DataTableRow) -> str: + """Finish a rename a crashed run started: point the row at "rdt_".""" + try: + with transaction.atomic(): + lock_resource(Resource.objects.get(pk=row.resource_id)) + updated = ResourceDataTable.objects.filter( + pk=row.row_id, table_name=row.table_name + ).update(table_name=data_table_name(row.row_id)) + except Exception as error: + logger.error("data_table_row_update_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + return "row_updated" if updated else "changed" + + +def rename_legacy_table(row: DataTableRow, unverified: bool = False) -> str: + """Rename the row's old table to "rdt_" and point the row at it. + + ``unverified`` stores the "unverified" mark in the table's comment, for a + long-named table kept without a rebuild (it may hold another resource's data). + + Holds the resource row, so an upload can't switch it meanwhile; a row an + upload already replaced is left alone ("changed"). Waits at most + RENAME_LOCK_TIMEOUT for readers ("skipped_lock"). The data database commits + first; if the row update then fails, a rerun finds the renamed table and + updates the row ("row_updated"). Returns "renamed" on success, "failed" + on any other error (logged). + """ + new_name = data_table_name(row.row_id) + try: + with transaction.atomic(): + resource = Resource.objects.select_related("dataset").get(pk=row.resource_id) + lock_resource(resource) + still_legacy = ResourceDataTable.objects.filter( + pk=row.row_id, table_name=row.table_name + ).exists() + if not still_legacy: + return "changed" + comment = describe_data_table(resource, built_at=row.created, unverified=unverified) + _rename_in_data_db(row.table_name, new_name, comment) + ResourceDataTable.objects.filter(pk=row.row_id).update(table_name=new_name) + except OperationalError as error: + if error_fields(error)["error_code"] == LOCK_NOT_AVAILABLE: + logger.warning("data_table_rename_skipped_lock", row_id=str(row.row_id)) + return "skipped_lock" + logger.error("data_table_rename_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + except Exception as error: + logger.error("data_table_rename_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + logger.info("data_table_renamed", row_id=str(row.row_id), table_name=new_name) + return "renamed" + + +def _rename_in_data_db(old_name: str, new_name: str, comment: str) -> None: + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL lock_timeout = {}").format(pg_sql.Literal(RENAME_LOCK_TIMEOUT)) + ) + cursor.execute( + pg_sql.SQL("ALTER TABLE {} RENAME TO {}").format( + pg_sql.Identifier(old_name), pg_sql.Identifier(new_name) + ) + ) + cursor.execute( + pg_sql.SQL("COMMENT ON TABLE {} IS {}").format( + pg_sql.Identifier(new_name), pg_sql.Literal(comment) + ) + ) + + +def file_problem(resource: Resource) -> Optional[str]: + """Why the resource's stored file can't become a table, or None if it can. + + "missing_file" (no file, or gone from storage) or "not_indexable" (a format + that never gets a table, e.g. the file was later replaced by a PDF). + Storage errors are raised. + """ + details = getattr(resource, "resourcefiledetails", None) + if not details or not details.file: + return "missing_file" + if not has_indexable_file(resource): + return "not_indexable" + if not details.file.storage.exists(details.file.name): + return "missing_file" + return None + + +def rebuild_decision(row: DataTableRow, shared: bool) -> str: + """What --apply will do with a long-named old table. + + "rebuild" when the stored file can become a table; otherwise + "unverified_rename" for a table only this row uses, or the file problem + ("missing_file" / "not_indexable") for a shared one, which is left alone. + "check_failed" (logged) when the check itself fails. + """ + try: + problem = file_problem(Resource.objects.get(pk=row.resource_id)) + except Exception as error: + logger.error("data_table_file_check_failed", row_id=str(row.row_id), **error_fields(error)) + return "check_failed" + if problem is None: + return "rebuild" + return problem if shared else "unverified_rename" + + +def reindex_decision(row: DataTableRow) -> str: + """What --apply will do with a row whose table is gone: "reindex" or the file problem.""" + try: + problem = file_problem(Resource.objects.get(pk=row.resource_id)) + except Exception as error: + logger.error("data_table_file_check_failed", row_id=str(row.row_id), **error_fields(error)) + return "check_failed" + return problem or "reindex" + + +def rebuild_from_file(row: DataTableRow, shared: bool) -> str: + """Carry out the rebuild decision for a long-named old table. + + Rebuilds go through the normal upload flow with no limit on reading time, + so the resource gets a fresh per-upload table and the old one is dropped + once no row uses it. Returns "rebuilt", "unverified" (renamed and marked), + the rename's other outcomes, a file problem, "check_failed" or "failed". + """ + decision = rebuild_decision(row, shared) + if decision == "unverified_rename": + outcome = rename_legacy_table(row, unverified=True) + return "unverified" if outcome == "renamed" else outcome + if decision != "rebuild": + return decision + return _index_without_time_limit(row) + + +def reindex_missing_table(row: DataTableRow) -> str: + """Re-index a row whose table is gone: "reindexed", a file problem, or a failure.""" + decision = reindex_decision(row) + if decision != "reindex": + return decision + outcome = _index_without_time_limit(row) + return "reindexed" if outcome == "rebuilt" else outcome + + +def _index_without_time_limit(row: DataTableRow) -> str: + try: + resource = Resource.objects.select_related("dataset").get(pk=row.resource_id) + return "rebuilt" if index_resource_data(resource, load_timeout=None) else "failed" + except Exception as error: + logger.error("data_table_rebuild_failed", row_id=str(row.row_id), **error_fields(error)) + return "failed" + + +def find_unverified(rows: List[DataTableRow], tables: List[DataDbTable]) -> List[DataTableRow]: + """Rows whose table carries the "unverified" mark — kept without a rebuild, needing a check.""" + marked = {table.name for table in tables if table_is_unverified(table.comment)} + return [ + row + for row in rows + if physical_table_name(row.table_name) in marked or data_table_name(row.row_id) in marked + ] + + +def access_models_touched(resource_ids: Iterable[Any]) -> List[str]: + """Access models whose column choices a rebuild resets (columns are recreated).""" + ids = ( + AccessModelResource.objects.filter(resource_id__in=list(resource_ids)) + .values_list("access_model_id", flat=True) + .distinct() + ) + return sorted(str(access_model_id) for access_model_id in ids) + + +def read_reviewed_orphans(path: str) -> Set[str]: + """The orphan table names listed in a report someone reviewed. + + Raises CommandError if the file can't be read or isn't a report, so a bad + path stops the run before it changes anything. + """ + try: + with open(path, encoding="utf-8") as report: + return {orphan["name"] for orphan in json.load(report)["orphans"]} + except (OSError, ValueError, KeyError, TypeError) as error: + raise CommandError(f"Can't read the reviewed report {path}: {error}") from error + + +def same_file(first: str, second: str) -> bool: + """Whether two paths name the same file.""" + return os.path.abspath(first) == os.path.abspath(second) + + +def drop_reviewed_orphans(reviewed: Set[str], current_orphans: List[DataDbTable]) -> List[str]: + """Drop tables that are both in the reviewed report and still orphans now.""" + dropped = [] + for table in current_orphans: + if table.name in reviewed and drop_data_table(table.name): + dropped.append(table.name) + return dropped + + +def write_report(path: str, report: Dict[str, Any]) -> None: + """Write the run's report as JSON (ids, names, outcomes and sizes; no data).""" + with open(path, "w", encoding="utf-8") as out: + json.dump(report, out, indent=2, default=str) + + +def record_outcome( + row: DataTableRow, action: str, outcome: str, shared: Optional[bool] = None +) -> Dict[str, Any]: + """Log one row's outcome as it happens, and return its line for the report.""" + entry: Dict[str, Any] = { + "row_id": str(row.row_id), + "resource_id": str(row.resource_id), + "table_name": row.table_name, + "action": action, + "outcome": outcome, + } + if shared is not None: + entry["shared"] = shared + logger.info( + "data_table_migration_row", + row_id=entry["row_id"], + resource_id=entry["resource_id"], + action=action, + outcome=outcome, + ) + return entry + + +def nothing_left_to_do(entries: List[Dict[str, Any]]) -> bool: + """Whether a rerun would change nothing: every row is migrated or needs a person.""" + return all(entry["outcome"] in SETTLED_OUTCOMES for entry in entries) + + +def build_report( + mode: str, + generated_at: datetime, + entries: Optional[List[Dict[str, Any]]] = None, + orphans: Optional[List[DataDbTable]] = None, + orphans_dropped: Optional[List[str]] = None, + access_models: Optional[List[str]] = None, + unverified: Optional[List[DataTableRow]] = None, + shared_groups: int = 0, + status: str = "complete", +) -> Dict[str, Any]: + """The run's JSON report: counts by outcome, per-row lines, orphans, unverified tables. + + Orphans carry their size and comment (resource, dataset, build time) so a + reviewer can tell what each is before approving its drop. + """ + entries = entries or [] + counts = Counter(entry["outcome"] for entry in entries) + return { + "mode": mode, + "status": status, + "generated_at": generated_at.isoformat(), + "counts": dict(sorted(counts.items())), + "shared_groups": shared_groups, + "rows": entries, + "needs_manual_fix": [e for e in entries if e["outcome"] in NEEDS_MANUAL_FIX], + "unverified_tables": [ + {"row_id": str(r.row_id), "resource_id": str(r.resource_id), "table_name": r.table_name} + for r in unverified or [] + ], + "orphans": [ + {"name": t.name, "size_bytes": t.size_bytes, "comment": t.comment} + for t in orphans or [] + ], + "orphans_dropped": orphans_dropped or [], + "access_models_touched": access_models or [], + } + + +def summary_lines(report: Dict[str, Any], report_path: str) -> List[str]: + """A short plain-text summary of a report, for the terminal and the logs.""" + orphan_bytes = sum(orphan["size_bytes"] for orphan in report["orphans"]) + lines = [f"{report['mode']}: report written to {report_path}"] + lines += [f" {outcome}: {count}" for outcome, count in report["counts"].items()] + lines.append(f" old tables shared by several rows: {report['shared_groups']}") + lines.append( + f" rows needing a manual fix (file missing or unusable): {len(report['needs_manual_fix'])}" + ) + lines.append( + f" unverified tables (may hold another resource's data): {len(report['unverified_tables'])}" + ) + lines.append(f" orphans listed: {len(report['orphans'])} ({orphan_bytes} bytes)") + lines.append(f" orphans dropped: {len(report['orphans_dropped'])}") + lines.append( + f" access models with column choices reset: {len(report['access_models_touched'])}" + ) + logger.info( + "data_table_migration_finished", + mode=report["mode"], + shared_groups=report["shared_groups"], + needs_manual_fix=len(report["needs_manual_fix"]), + unverified_tables=len(report["unverified_tables"]), + orphans_listed=len(report["orphans"]), + orphans_dropped=len(report["orphans_dropped"]), + **report["counts"], + ) + return lines diff --git a/api/utils/data_table_names.py b/api/utils/data_table_names.py new file mode 100644 index 00000000..7be81964 --- /dev/null +++ b/api/utils/data_table_names.py @@ -0,0 +1,44 @@ +"""How tables in the data database are named. + +Every tabular upload gets its own table, named "rdt_" plus the 32-character +hex of its ResourceDataTable row id (36 characters in all). The name never +depends on a resource or dataset title, so renames don't touch it and two +uploads can never share one. + +Postgres keeps only the first 63 bytes of any name. Names from before this +rule were built from titles (up to 110 characters), so two of them can map to +the same physical table; ``physical_table_name`` gives the name Postgres +actually stored, for the checks that care. + +No imports from models: the ResourceDataTable model uses this module. +Docs: ./data_indexing_architecture.md +""" + +import uuid + +DATA_TABLE_PREFIX = "rdt_" +MAX_IDENTIFIER_BYTES = 63 +# A cut name is at least this long: a cut never drops more than a 4-byte character's first 3 bytes. +SHORTEST_CUT_NAME_BYTES = MAX_IDENTIFIER_BYTES - 3 + + +def data_table_name(row_id: uuid.UUID) -> str: + """The table name for a ResourceDataTable row: "rdt_" + its id in lowercase hex.""" + return f"{DATA_TABLE_PREFIX}{row_id.hex}" + + +def physical_table_name(name: str) -> str: + """The name Postgres stores for ``name``: its first 63 bytes, never half a character.""" + encoded = name.encode("utf-8") + if len(encoded) <= MAX_IDENTIFIER_BYTES: + return name + return encoded[:MAX_IDENTIFIER_BYTES].decode("utf-8", errors="ignore") + + +def may_share_physical_table(name: str) -> bool: + """Whether another, longer name could be cut to the same physical table as this one. + + Only names at least as long as the shortest possible cut can; "rdt_" names + (36 bytes) never can. + """ + return len(name.encode("utf-8")) >= SHORTEST_CUT_NAME_BYTES diff --git a/api/utils/data_tables.py b/api/utils/data_tables.py new file mode 100644 index 00000000..9b810f7e --- /dev/null +++ b/api/utils/data_tables.py @@ -0,0 +1,479 @@ +"""Helpers for copying uploaded files into tables in the data database. + +The upload flow (``index_resource_data`` in ./data_indexing.py) reads a +resource's file, builds a brand-new table for it, then switches the resource's +ResourceDataTable row over to that table in one main-database commit. Old tables +are dropped after the commit by the row's delete receiver +(api/signals/data_table_signals.py). Each helper here does one step of that. +The last few (listing the data database's tables, the "is this really a +separate database?" check) serve the ``migrate_data_table_names`` command. + +Tables live in the separate ``data_db`` database (its statements autocommit); +the rows that point at them (ResourceDataTable, ResourceSchema) live in the +main database. New SQL composes names and comments with psycopg2's +``sql.Identifier`` / ``sql.Literal``, never with string formatting. + +Logs carry ids, counts and error codes only — never file contents or values. +Docs: ./data_indexing_architecture.md +""" + +import json +import os +import signal +import threading +import traceback +from collections import Counter +from datetime import datetime +from io import StringIO +from typing import Any, Dict, List, NamedTuple, Optional, Set + +import pandas as pd +import structlog +from django.apps import apps +from django.db import connections, transaction +from psycopg2 import sql as pg_sql # type: ignore[import-untyped] + +from api.models.Resource import Resource, ResourceDataTable, ResourceFileDetails +from api.models.ResourceSchema import ResourceSchema +from api.utils.data_table_names import ( + data_table_name, + may_share_physical_table, + physical_table_name, +) +from api.utils.file_utils import load_tabular_data, local_copy + +logger = structlog.get_logger("dataspace.data_tables") + +DATA_DB = "data_db" # the connection name in settings.DATABASES + +# Formats copied into a table (and so readable through the data API). +INDEXED_FORMATS = {"csv", "xls", "xlsx", "ods", "parquet", "feather", "json", "tsv"} + +# How long a drop waits for readers holding the table before giving up. +DROP_LOCK_TIMEOUT = "5s" + +# How long reading a file may take, where the platform lets us stop it. +LOAD_TIMEOUT_SECONDS = 60 + + +class FileIdentity(NamedTuple): + """Which stored file a build read: its storage name and SHA-256.""" + + name: str + sha256: Optional[str] + + +class LoadedFile(NamedTuple): + """A resource's file read into a frame, with the identity of that file.""" + + frame: pd.DataFrame + file: FileIdentity + + +class NewerFileSaved(Exception): + """Another upload saved a newer file while this one was being built.""" + + +def get_sql_type(pandas_dtype: str) -> str: + """The Postgres column type for a pandas dtype name (TEXT when unsure).""" + if "int" in pandas_dtype: + return "BIGINT" + if "float" in pandas_dtype: + return "DOUBLE PRECISION" + if "datetime" in pandas_dtype: + return "TIMESTAMP" + if "bool" in pandas_dtype: + return "BOOLEAN" + return "TEXT" + + +_FIELD_FORMATS = { + "BIGINT": "INTEGER", + "DOUBLE PRECISION": "NUMBER", + "TIMESTAMP": "DATE", + "BOOLEAN": "BOOLEAN", +} + + +def error_fields(error: BaseException) -> Dict[str, Any]: + """Log fields for an error: its class, Postgres error code and where it was raised. + + Never its message: database messages quote the offending value (a COPY + error names the cell). ``where`` is the innermost frame in our own code + ("data_tables.py:231 build_data_table"), which carries no data. + """ + cause = error.__cause__ or error + frames = traceback.extract_tb(error.__traceback__) + ours = [f for f in frames if f"{os.sep}api{os.sep}" in f.filename] or frames + where = ( + f"{os.path.basename(ours[-1].filename)}:{ours[-1].lineno} {ours[-1].name}" if ours else None + ) + return { + "error_type": type(error).__name__, + "error_code": getattr(cause, "pgcode", None), + "where": where, + } + + +def is_inside_transaction() -> bool: + """Whether the main database is already inside a transaction (an atomic block).""" + return bool(transaction.get_connection().in_atomic_block) + + +def has_indexable_file(resource: Resource) -> bool: + """Whether the resource has a stored file in a format that gets a table.""" + file_details = getattr(resource, "resourcefiledetails", None) + if not file_details or not file_details.file: + logger.info("data_index_skipped_no_file", resource_id=str(resource.id)) + return False + file_format = (file_details.format or "").lower() + if file_format not in INDEXED_FORMATS: + logger.info("data_index_skipped_format", resource_id=str(resource.id), format=file_format) + return False + return True + + +def load_resource_frame( + resource: Resource, timeout: Optional[int] = LOAD_TIMEOUT_SECONDS +) -> Optional[LoadedFile]: + """Read the resource's stored file into a frame and note which file it was. + + Column names are cleaned the way tables have always needed them: double + quotes removed, line breaks and tabs turned into spaces, and names that + then collide get "_1", "_2", ... suffixes. + + Returns None when the file can't be read, takes longer than ``timeout`` + seconds (None: no limit), or has no rows (each case logged). + """ + file_details = resource.resourcefiledetails + identity = FileIdentity(file_details.file.name, file_details.sha256) + try: + with local_copy(file_details.file) as path: + frame = _load_with_timeout(path, file_details.format.lower(), timeout) + except Exception as error: + logger.error("data_index_load_failed", resource_id=str(resource.id), **error_fields(error)) + return None + if frame is None: + logger.error("data_index_load_failed", resource_id=str(resource.id), error_type="NoFrame") + return None + if frame.empty: + logger.info("data_index_skipped_empty", resource_id=str(resource.id)) + return None + frame.columns = pd.Index(_clean_column_names(list(frame.columns))) + return LoadedFile(frame, identity) + + +def _load_with_timeout( + path: str, file_format: str, timeout: Optional[int] +) -> Optional[pd.DataFrame]: + # SIGALRM only works in a Unix process's main thread; elsewhere (a server + # worker thread) the file loads without a limit, as it always has. + in_main_thread = threading.current_thread() is threading.main_thread() + if timeout is None or not in_main_thread or not hasattr(signal, "SIGALRM"): + return load_tabular_data(path, file_format) + + def on_alarm(signum: int, frame: Any) -> None: + raise TimeoutError(f"loading took longer than {timeout} seconds") + + previous = signal.signal(signal.SIGALRM, on_alarm) + signal.alarm(timeout) + try: + return load_tabular_data(path, file_format) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous) + + +def _clean_column_names(columns: List[Any]) -> List[str]: + # TODO: Postgres keeps only the first 63 bytes of a column name, so two headers that + # differ only after that (about 21 Devanagari characters) become one column and the + # file never gets a table. Known, not fixed yet: data_indexing_architecture.md, Limitations. + cleaned = [str(c).replace('"', "").replace("\n", " ").replace("\t", " ") for c in columns] + seen: Counter = Counter() + unique = [] + for name in cleaned: + unique.append(f"{name}_{seen[name]}" if seen[name] else name) + seen[name] += 1 + return unique + + +def stored_file_identity(resource: Resource) -> Optional[FileIdentity]: + """The resource's current stored file, read fresh from the database.""" + row = ( + ResourceFileDetails.objects.filter(resource_id=resource.pk) + .values_list("file", "sha256") + .first() + ) + return FileIdentity(*row) if row else None + + +def new_data_table_row(resource: Resource) -> ResourceDataTable: + """An unsaved ResourceDataTable row for this upload, already carrying its table name.""" + row = ResourceDataTable(resource=resource) + row.table_name = data_table_name(row.id) + return row + + +def describe_data_table(resource: Resource, built_at: datetime, unverified: bool = False) -> str: + """The comment stored on a new table, so people browsing the data database know what it holds. + + JSON with the resource id and name, the dataset title and the build time; + the migration command reads ``built_at`` to leave freshly built tables alone. + ``unverified`` marks an old table the migration kept without rebuilding it + (its file was missing), which may hold another resource's data; the mark + stays until the resource's next upload replaces the table. + """ + description: Dict[str, Any] = { + "resource_id": str(resource.id), + "resource": resource.name, + "dataset": resource.dataset.title, + "built_at": built_at.isoformat(), + } + if unverified: + description["unverified"] = True + return json.dumps(description, ensure_ascii=False) + + +def build_data_table(table_name: str, frame: pd.DataFrame, comment: str) -> None: + """Create a table holding every row of the frame, in one data-database transaction. + + Creates the table, copies the rows in through a temporary table, and + stores the comment. Raises on any failure, and then nothing is left + behind: a killed process or a bad value mid-copy rolls the whole table back. + """ + table = pg_sql.Identifier(table_name) + staging = pg_sql.Identifier(f"tmp_{table_name}") + columns = pg_sql.SQL(", ").join( + pg_sql.SQL("{} {}").format( + pg_sql.Identifier(str(col)), pg_sql.SQL(get_sql_type(str(frame[col].dtype))) + ) + for col in frame.columns + ) + rows = StringIO() + frame.to_csv(rows, index=False, header=False) + rows.seek(0) + + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute(pg_sql.SQL("CREATE TABLE {} ({})").format(table, columns)) + cursor.execute( + pg_sql.SQL("CREATE TEMP TABLE {} (LIKE {}) ON COMMIT DROP").format(staging, table) + ) + cursor.copy_expert(pg_sql.SQL("COPY {} FROM STDIN WITH CSV").format(staging), rows) + cursor.execute(pg_sql.SQL("INSERT INTO {} SELECT * FROM {}").format(table, staging)) + cursor.execute( + pg_sql.SQL("COMMENT ON TABLE {} IS {}").format(table, pg_sql.Literal(comment)) + ) + + +def lock_resource(resource: Resource) -> None: + """Lock the resource's row until the surrounding transaction ends. + + Two uploads of one resource then switch one after the other. A no-op on + SQLite (tests); Postgres in production. + """ + Resource.objects.select_for_update().only("id").get(pk=resource.pk) + + +def retire_data_table_row(resource: Resource) -> Optional[datetime]: + """Delete the resource's current ResourceDataTable row, returning when it was created. + + Its table is dropped by the delete receiver once the surrounding + transaction commits. Returns None when the resource had no row. + """ + old = ResourceDataTable.objects.filter(resource_id=resource.pk).first() + if old is None: + return None + created = old.created + old.delete() + return created + + +def save_data_table_row(row: ResourceDataTable, created: Optional[datetime]) -> None: + """Save the new row, keeping the retired row's creation time when there was one. + + The data API picks a dataset's default table by creation time, so a + re-upload keeps the dataset's default where it was. + """ + row.save(force_insert=True) + if created is not None: + ResourceDataTable.objects.filter(pk=row.pk).update(created=created) + row.created = created + + +def rebuild_resource_schema(resource: Resource, frame: pd.DataFrame) -> None: + """Replace the resource's column rows with one per frame column. + + Each column's type comes from the frame. Descriptions people wrote are + carried over to columns with the same name; new columns get + "Description of column ". + + Errors escape: inside the switch transaction a swallowed database error + would silently roll the whole switch back. A header longer than the column + row allows is refused up front (ValueError naming its position), the same + outcome Postgres would give, on every database. + """ + max_length = ResourceSchema._meta.get_field("field_name").max_length or 255 + for position, column in enumerate(frame.columns, start=1): + if len(str(column)) > max_length: + raise ValueError(f"column {position} header is longer than {max_length} characters") + + existing = ResourceSchema.objects.filter(resource=resource) + descriptions = { + name: description + for name, description in existing.values_list("field_name", "description") + if description is not None + } + existing.delete() + for column in frame.columns: + sql_type = get_sql_type(str(frame[column].dtype)) + ResourceSchema.objects.create( + resource=resource, + field_name=column, + format=_FIELD_FORMATS.get(sql_type, "STRING"), + description=descriptions.get(column, f"Description of column {column}"), + ) + + +def data_table_row_exists(row_id: Any) -> bool: + """Whether a ResourceDataTable row with this id is saved.""" + return ResourceDataTable.objects.filter(pk=row_id).exists() + + +def drop_data_table(table_name: str) -> bool: + """Drop a table from the data database; never raises. + + Waits at most DROP_LOCK_TIMEOUT for readers holding the table. A failure + is logged with the table name and swallowed — the table is left behind as + an orphan for ``migrate_data_table_names`` to report. Returns whether the + drop went through (a table that was already gone counts as dropped). + """ + try: + with transaction.atomic(using=DATA_DB), connections[DATA_DB].cursor() as cursor: + cursor.execute( + pg_sql.SQL("SET LOCAL lock_timeout = {}").format(pg_sql.Literal(DROP_LOCK_TIMEOUT)) + ) + cursor.execute( + pg_sql.SQL("DROP TABLE IF EXISTS {}").format(pg_sql.Identifier(table_name)) + ) + except Exception as error: + logger.error("data_table_drop_failed", table_name=table_name, **error_fields(error)) + return False + logger.info("data_table_dropped", table_name=table_name) + return True + + +def drop_unswitched_table(row: ResourceDataTable) -> None: + """Drop the table of a build whose switch wasn't confirmed, unless its row was saved after all. + + The switch can commit and still not be confirmed: a run stopped (Ctrl+C) + while the old table's after-commit drop was waiting, or a commit that went + through but reported an error. So the row is checked first; if it exists, + the table is the resource's live data and stays. If the check itself fails, + the table is left for the migration command's orphan audit. Never raises + an Exception. + """ + try: + saved = data_table_row_exists(row.pk) + except Exception as error: + logger.error( + "data_index_switch_check_failed", table_name=row.table_name, **error_fields(error) + ) + return + if saved: + logger.warning("data_index_switch_saved_after_all", table_name=row.table_name) + return + drop_data_table(row.table_name) + + +def drop_data_table_if_unshared(table_name: str, row_id: Any) -> None: + """Drop a deleted row's table, unless another row still uses the same physical table. + + Only names from before per-upload tables can share one (Postgres cut two + long names to the same 63 bytes); for those, the table stays until its last + row is gone. Query: ``table_name LIKE '%'`` on the unique + table_name index — a sequential scan on some collations, over one row per + tabular resource. Never raises. + """ + try: + shared = may_share_physical_table(table_name) and _another_row_uses(table_name, row_id) + except Exception as error: + logger.error("data_table_drop_failed", table_name=table_name, **error_fields(error)) + return + if shared: + logger.info("data_table_drop_skipped_shared", table_name=table_name, row_id=str(row_id)) + return + drop_data_table(table_name) + + +def _another_row_uses(table_name: str, row_id: Any) -> bool: + physical = physical_table_name(table_name) + candidates = ( + ResourceDataTable.objects.filter(table_name__startswith=physical) + .exclude(pk=row_id) + .values_list("table_name", flat=True) + ) + return any(physical_table_name(name) == physical for name in candidates) + + +class DataDbTable(NamedTuple): + """A table in the data database's public schema.""" + + name: str + size_bytes: int + comment: Optional[str] + + +class DataDbNotSeparate(Exception): + """The data database looks like (or holds tables of) the main database.""" + + +def list_data_db_tables() -> List[DataDbTable]: + """Every ordinary table in the data database's public schema, with its size and comment.""" + with connections[DATA_DB].cursor() as cursor: + cursor.execute( + "SELECT c.relname, pg_total_relation_size(c.oid), obj_description(c.oid, 'pg_class') " + "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'public' AND c.relkind = 'r' ORDER BY c.relname" + ) + return [DataDbTable(name, int(size), comment) for name, size, comment in cursor.fetchall()] + + +def django_table_names() -> Set[str]: + """The tables Django's own models use (plus its migration log).""" + models_tables = {m._meta.db_table for m in apps.get_models(include_auto_created=True)} + return models_tables | {"django_migrations"} + + +def assert_data_db_is_separate() -> None: + """Raise DataDbNotSeparate unless the data database is clearly not the main one. + + Refused when both connections name the same host, port and database, or + when the data database holds ``django_migrations`` or any table a Django + model uses — either way, every application table would look like an + unused data table to the orphan sweep. + """ + main = connections["default"].settings_dict + data = connections[DATA_DB].settings_dict + if all( + str(main.get(key) or "") == str(data.get(key) or "") for key in ("HOST", "PORT", "NAME") + ): + raise DataDbNotSeparate("the data database connection points at the main database") + found = sorted(django_table_names() & {table.name for table in list_data_db_tables()}) + if found: + raise DataDbNotSeparate(f"the data database holds Django tables: {', '.join(found[:5])}") + + +def table_built_at(comment: Optional[str]) -> Optional[datetime]: + """When a per-upload table was built, read from its comment (None if unreadable).""" + try: + return datetime.fromisoformat(json.loads(comment or "")["built_at"]) + except (ValueError, KeyError, TypeError): + return None + + +def table_is_unverified(comment: Optional[str]) -> bool: + """Whether a table's comment carries the migration's "unverified" mark.""" + try: + return json.loads(comment or "").get("unverified") is True + except (ValueError, AttributeError): + return False diff --git a/api/utils/dataset_access.py b/api/utils/dataset_access.py new file mode 100644 index 00000000..e8eb8943 --- /dev/null +++ b/api/utils/dataset_access.py @@ -0,0 +1,145 @@ +"""Who may see or change a dataset and the things hanging off it. + +Resources, charts, chart images and access models have no rights of their own: +anyone who may change the dataset may change them, and they are visible to +whoever may see the dataset. The organisation is always the dataset's own; +nothing here reads the request's organisation header. +""" + +from typing import Any, Optional + +import structlog +from django.db.models import Q +from strawberry.permission import BasePermission +from strawberry.types import Info + +from api.models import Dataset +from api.utils.enums import DatasetStatus +from authorization.models import DatasetPermission, OrganizationMembership + +logger = structlog.get_logger(__name__) + +# Organisation roles that may edit the organisation's datasets, by name. +DATASET_EDITOR_ROLE_NAMES = ["admin", "editor", "owner"] + + +def can_change_dataset(user: Any, dataset: Dataset) -> bool: + """The one dataset edit rule, also used by UpdateDatasetPermission.""" + if not user or not user.is_authenticated: + return False + if user.is_superuser or dataset.user_id == user.id: + return True + if ( + dataset.organization_id + and OrganizationMembership.objects.filter( + user=user, + organization_id=dataset.organization_id, + role__name__in=DATASET_EDITOR_ROLE_NAMES, + ).exists() + ): + return True + dataset_perm = ( + DatasetPermission.objects.filter(user=user, dataset=dataset).select_related("role").first() + ) + return bool( + dataset_perm + and dataset_perm.role.can_change + and dataset.status == DatasetStatus.DRAFT.value + ) + + +def can_view_dataset(user: Any, dataset: Dataset) -> bool: + """Published datasets are public; otherwise any one grant is enough.""" + if dataset.status == DatasetStatus.PUBLISHED.value: + return True + if not user or not user.is_authenticated: + return False + if user.is_superuser or dataset.user_id == user.id: + return True + if ( + dataset.organization_id + and OrganizationMembership.objects.filter( + user=user, organization_id=dataset.organization_id, role__can_view=True + ).exists() + ): + return True + if DatasetPermission.objects.filter(user=user, dataset=dataset, role__can_view=True).exists(): + return True + return can_change_dataset(user, dataset) + + +def viewable_datasets_q(user: Any, prefix: str = "") -> Q: + """`can_view_dataset` as a filter, for lists. `prefix` reaches the dataset, e.g. "dataset__".""" + published = Q(**{f"{prefix}status": DatasetStatus.PUBLISHED.value}) + if not user or not user.is_authenticated: + return published + if user.is_superuser: + return Q() + orgs = OrganizationMembership.objects.filter(user=user).filter( + Q(role__can_view=True) | Q(role__name__in=DATASET_EDITOR_ROLE_NAMES) + ) + shared = DatasetPermission.objects.filter(user=user, role__can_view=True) + draft_editors = DatasetPermission.objects.filter( + user=user, role__can_change=True, dataset__status=DatasetStatus.DRAFT.value + ) + return ( + published + | Q(**{f"{prefix}user_id": user.id}) + | Q(**{f"{prefix}organization_id__in": orgs.values("organization_id")}) + | Q(**{f"{prefix}id__in": shared.values("dataset_id")}) + | Q(**{f"{prefix}id__in": draft_editors.values("dataset_id")}) + ) + + +def can_list_organization(user: Any, organization: Any) -> bool: + """Whether this user may list an organisation's datasets, drafts included.""" + if not user or not user.is_authenticated: + return False + if user.is_superuser: + return True + return OrganizationMembership.objects.filter( + user=user, organization=organization, role__can_view=True + ).exists() + + +def log_access_denied( + info: Info, user: Any, reason: str, dataset: Optional[Dataset] = None +) -> bool: + """Record one warning for a refused request, and return False.""" + logger.warning( + "dataset access denied", + operation=info.field_name, + user_id=getattr(user, "id", None), + reason=reason, + dataset_id=str(dataset.id) if dataset else None, + organization_id=dataset.organization_id if dataset else None, + ) + return False + + +class ChangeDatasetPartPermission(BasePermission): + """Base for mutations on something that belongs to a dataset. + + Subclasses return every dataset the call touches (the one the object is in + now and, when it is being moved, the one it goes to); the user needs edit + rights on all of them. A missing ID is refused the same way, so the answer + does not reveal whether an ID exists. + """ + + message = "You don't have permission to change this dataset" + + def datasets(self, kwargs: Any) -> Optional[list]: + """The datasets the call touches, or None if any ID does not exist.""" + raise NotImplementedError + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = getattr(info.context, "user", None) + if not user or not user.is_authenticated: + return log_access_denied(info, user, "not_signed_in") + datasets = self.datasets(kwargs) + if not datasets: + return log_access_denied(info, user, "target_not_found") + for dataset in datasets: + if not can_change_dataset(user, dataset): + return log_access_denied(info, user, "not_allowed", dataset) + return True diff --git a/api/utils/django_utils.py b/api/utils/django_utils.py index f9220439..2c38403f 100644 --- a/api/utils/django_utils.py +++ b/api/utils/django_utils.py @@ -1,46 +1,12 @@ -from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast -from django.core.exceptions import ValidationError -from django.db import IntegrityError, models, transaction +from django.db import models from django.db.models import Field, Model, QuerySet T = TypeVar("T", bound=models.Model) GraphQLType = TypeVar("GraphQLType") -def retry_on_slug_collision( - save_fn: Callable[[], None], - disambiguate: Callable[[int], None], - max_attempts: int = 5, -) -> None: - """Retry a model save that failed on a title/slug uniqueness collision. - - Auto-generated titles like "New dataset 17 Sep 2026 - 09:43:04" only have - one-second resolution, so two records created in the same second collide - on the unique slug (and, for models where title is also unique, on title - too). On each retry `disambiguate(attempt)` should mutate the instance's - title/slug to a new candidate (e.g. append " (n)") before `save_fn` is - called again. Each attempt runs in its own savepoint so a failed attempt - doesn't poison the caller's outer transaction. - - ponytail: catches ValidationError broadly (not just the title/slug keys) - so an unrelated validation failure retries pointlessly a few times before - surfacing the same error - upgrade to inspecting message_dict if that - noise ever matters. - """ - attempt = 0 - while True: - try: - with transaction.atomic(): - save_fn() - return - except (IntegrityError, ValidationError): - attempt += 1 - if attempt >= max_attempts: - raise - disambiguate(attempt) - - def get_graphql_type_fields_name(type_: Type[GraphQLType]) -> Dict[str, Any]: """Get field names from a GraphQL type.""" fields = type_.__dict__.get("__dataclass_fields__", {}) diff --git a/api/utils/enums.py b/api/utils/enums.py index 49e1a610..686153fe 100644 --- a/api/utils/enums.py +++ b/api/utils/enums.py @@ -179,11 +179,131 @@ class DatasetAccessType(models.TextChoices): class DatasetLicense(models.TextChoices): - GOVERNMENT_OPEN_DATA_LICENSE = "GOVERNMENT_OPEN_DATA_LICENSE" - CC_BY_4_0_ATTRIBUTION = "CC_BY_4_0_ATTRIBUTION" - CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE = "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE" - OPEN_DATA_COMMONS_BY_ATTRIBUTION = "OPEN_DATA_COMMONS_BY_ATTRIBUTION" - OPEN_DATABASE_LICENSE = "OPEN_DATABASE_LICENSE" + """One value per row of contracts/licenses.csv (matched by its dataspace_enum alt code).""" + + GOVERNMENT_OPEN_DATA_LICENSE = ( + "GOVERNMENT_OPEN_DATA_LICENSE", + "Government Open Data License – India", + ) + CC_BY_4_0_ATTRIBUTION = "CC_BY_4_0_ATTRIBUTION", "Creative Commons Attribution 4.0" + CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE = ( + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "Creative Commons Attribution-ShareAlike 4.0", + ) + OPEN_DATA_COMMONS_BY_ATTRIBUTION = ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "Open Data Commons Attribution License 1.0", + ) + OPEN_DATABASE_LICENSE = "OPEN_DATABASE_LICENSE", "Open Database License 1.0" + CC_PDM_1_0 = "CC_PDM_1_0", "Creative Commons Public Domain Mark 1.0 Universal" + CC_BY_1_0 = "CC_BY_1_0", "Creative Commons Attribution 1.0 Generic" + CC_BY_2_0 = "CC_BY_2_0", "Creative Commons Attribution 2.0 Generic" + CC_BY_2_5_AU = "CC_BY_2_5_AU", "Creative Commons Attribution 2.5 Australia" + CC_BY_2_5 = "CC_BY_2_5", "Creative Commons Attribution 2.5 Generic" + CC_BY_3_0_AU = "CC_BY_3_0_AU", "Creative Commons Attribution 3.0 Australia" + CC_BY_3_0_AT = "CC_BY_3_0_AT", "Creative Commons Attribution 3.0 Austria" + CC_BY_3_0_DE = "CC_BY_3_0_DE", "Creative Commons Attribution 3.0 Germany" + CC_BY_3_0_IGO = "CC_BY_3_0_IGO", "Creative Commons Attribution 3.0 IGO" + CC_BY_3_0_NL = "CC_BY_3_0_NL", "Creative Commons Attribution 3.0 Netherlands" + CC_BY_3_0_US = "CC_BY_3_0_US", "Creative Commons Attribution 3.0 United States" + CC_BY_3_0 = "CC_BY_3_0", "Creative Commons Attribution 3.0 Unported" + CC_BY_ND_1_0 = "CC_BY_ND_1_0", "Creative Commons Attribution No Derivatives 1.0 Generic" + CC_BY_ND_2_0 = "CC_BY_ND_2_0", "Creative Commons Attribution No Derivatives 2.0 Generic" + CC_BY_ND_2_5 = "CC_BY_ND_2_5", "Creative Commons Attribution No Derivatives 2.5 Generic" + CC_BY_ND_3_0_DE = "CC_BY_ND_3_0_DE", "Creative Commons Attribution No Derivatives 3.0 Germany" + CC_BY_ND_3_0 = "CC_BY_ND_3_0", "Creative Commons Attribution No Derivatives 3.0 Unported" + CC_BY_ND_4_0 = "CC_BY_ND_4_0", "Creative Commons Attribution No Derivatives 4.0 International" + CC_BY_NC_1_0 = "CC_BY_NC_1_0", "Creative Commons Attribution Non Commercial 1.0 Generic" + CC_BY_NC_2_0 = "CC_BY_NC_2_0", "Creative Commons Attribution Non Commercial 2.0 Generic" + CC_BY_NC_2_5 = "CC_BY_NC_2_5", "Creative Commons Attribution Non Commercial 2.5 Generic" + CC_BY_NC_3_0_DE = "CC_BY_NC_3_0_DE", "Creative Commons Attribution Non Commercial 3.0 Germany" + CC_BY_NC_3_0_IGO = "CC_BY_NC_3_0_IGO", "Creative Commons Attribution Non Commercial 3.0 IGO" + CC_BY_NC_3_0 = "CC_BY_NC_3_0", "Creative Commons Attribution Non Commercial 3.0 Unported" + CC_BY_NC_4_0 = "CC_BY_NC_4_0", "Creative Commons Attribution Non Commercial 4.0 International" + CC_BY_NC_ND_1_0 = ( + "CC_BY_NC_ND_1_0", + "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + ) + CC_BY_NC_ND_2_0 = ( + "CC_BY_NC_ND_2_0", + "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + ) + CC_BY_NC_ND_2_5 = ( + "CC_BY_NC_ND_2_5", + "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + ) + CC_BY_NC_ND_3_0_DE = ( + "CC_BY_NC_ND_3_0_DE", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + ) + CC_BY_NC_ND_3_0_IGO = ( + "CC_BY_NC_ND_3_0_IGO", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + ) + CC_BY_NC_ND_3_0 = ( + "CC_BY_NC_ND_3_0", + "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + ) + CC_BY_NC_ND_4_0 = ( + "CC_BY_NC_ND_4_0", + "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + ) + CC_BY_NC_SA_1_0 = ( + "CC_BY_NC_SA_1_0", + "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + ) + CC_BY_NC_SA_2_0_UK = ( + "CC_BY_NC_SA_2_0_UK", + "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + ) + CC_BY_NC_SA_2_0 = ( + "CC_BY_NC_SA_2_0", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + ) + CC_BY_NC_SA_2_0_DE = ( + "CC_BY_NC_SA_2_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + ) + CC_BY_NC_SA_2_5 = ( + "CC_BY_NC_SA_2_5", + "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + ) + CC_BY_NC_SA_3_0_DE = ( + "CC_BY_NC_SA_3_0_DE", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + ) + CC_BY_NC_SA_3_0_IGO = ( + "CC_BY_NC_SA_3_0_IGO", + "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + ) + CC_BY_NC_SA_3_0 = ( + "CC_BY_NC_SA_3_0", + "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + ) + CC_BY_NC_SA_4_0 = ( + "CC_BY_NC_SA_4_0", + "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + ) + CC_BY_SA_1_0 = "CC_BY_SA_1_0", "Creative Commons Attribution Share Alike 1.0 Generic" + CC_BY_SA_2_0_UK = ( + "CC_BY_SA_2_0_UK", + "Creative Commons Attribution Share Alike 2.0 England and Wales", + ) + CC_BY_SA_2_0 = "CC_BY_SA_2_0", "Creative Commons Attribution Share Alike 2.0 Generic" + CC_BY_SA_2_1_JP = "CC_BY_SA_2_1_JP", "Creative Commons Attribution Share Alike 2.1 Japan" + CC_BY_SA_2_5 = "CC_BY_SA_2_5", "Creative Commons Attribution Share Alike 2.5 Generic" + CC_BY_SA_3_0_AT = "CC_BY_SA_3_0_AT", "Creative Commons Attribution Share Alike 3.0 Austria" + CC_BY_SA_3_0_DE = "CC_BY_SA_3_0_DE", "Creative Commons Attribution Share Alike 3.0 Germany" + CC_BY_SA_3_0 = "CC_BY_SA_3_0", "Creative Commons Attribution Share Alike 3.0 Unported" + CC_BY_NC_SA_2_0_FR = ( + "CC_BY_NC_SA_2_0_FR", + "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + ) + CC_BY_SA_3_0_IGO = "CC_BY_SA_3_0_IGO", "Creative Commons Attribution-ShareAlike 3.0 IGO" + CC_PDDC = "CC_PDDC", "Creative Commons Public Domain Dedication and Certification" + CC_SA_1_0 = "CC_SA_1_0", "Creative Commons Share Alike 1.0 Generic" + CC0_1_0 = "CC0_1_0", "Creative Commons Zero v1.0 Universal" + PDDL_1_0 = "PDDL_1_0", "Open Data Commons Public Domain Dedication & License 1.0" class UseCaseStatus(models.TextChoices): @@ -197,6 +317,11 @@ class PublicationStatus(models.TextChoices): PUBLISHED = "PUBLISHED" +class EventStatus(models.TextChoices): + DRAFT = "DRAFT" + PUBLISHED = "PUBLISHED" + + class PublicationBlockType(models.TextChoices): FILE = "FILE" YOUTUBE = "YOUTUBE" diff --git a/api/utils/file_paths.py b/api/utils/file_paths.py index aa1ecaa2..7cddf28b 100644 --- a/api/utils/file_paths.py +++ b/api/utils/file_paths.py @@ -2,6 +2,9 @@ import uuid from typing import Any +# Names are relative to the storage, which already prefixes every key with +# /public or /private; see api/utils/storage.py. + def _organization_directory_path(org: Any, filename: str) -> str: """ @@ -11,7 +14,7 @@ def _organization_directory_path(org: Any, filename: str) -> str: org_name = org.name _, extension = os.path.splitext(filename) - return f"files/public/organizations/{org_name}/{extension[1:]}/{filename}" + return f"organizations/{org_name}/{extension[1:]}/{filename}" def _chart_image_directory_path(chart: Any, filename: str) -> str: @@ -22,7 +25,7 @@ def _chart_image_directory_path(chart: Any, filename: str) -> str: chart_name = chart.name _, extension = os.path.splitext(filename) - return f"files/chart_images/{chart_name}/{extension[1:]}/{filename}" + return f"chart_images/{chart_name}/{extension[1:]}/{filename}" def _dataspace_directory_path(ds: Any, filename: str) -> str: @@ -33,7 +36,7 @@ def _dataspace_directory_path(ds: Any, filename: str) -> str: ds_name = ds.name _, extension = os.path.splitext(filename) - return f"files/public/dataspace/{ds_name}/{extension[1:]}/{filename}" + return f"dataspace/{ds_name}/{extension[1:]}/{filename}" def _use_case_directory_path(uc: Any, filename: str) -> str: @@ -44,7 +47,7 @@ def _use_case_directory_path(uc: Any, filename: str) -> str: uc_name = uc.title _, extension = os.path.splitext(filename) - return f"files/use_case/{uc_name}/logo/{filename}" + return f"use_case/{uc_name}/logo/{filename}" def _user_profile_image_directory_path(user: Any, filename: str) -> str: @@ -54,7 +57,7 @@ def _user_profile_image_directory_path(user: Any, filename: str) -> str: """ user_name = user.keycloak_id _, extension = os.path.splitext(filename) - return f"files/user_profile/{user_name}/{extension[1:]}/{filename}" + return f"user_profile/{user_name}/{extension[1:]}/{filename}" def _organization_file_directory_path(org: Any, filename: str) -> str: @@ -65,7 +68,7 @@ def _organization_file_directory_path(org: Any, filename: str) -> str: org_name = org.name _, extension = os.path.splitext(filename) - return f"files/resources/{org_name}/sample_data/{extension[1:]}/{filename}" + return f"resources/{org_name}/sample_data/{extension[1:]}/{filename}" def _catalog_directory_path(catalog: Any, filename: str) -> str: @@ -74,7 +77,7 @@ def _catalog_directory_path(catalog: Any, filename: str) -> str: """ catalog_name = catalog.name _, extension = os.path.splitext(filename) - return f"files/catalog/{catalog_name}/{extension[1:]}/{filename}" + return f"catalog/{catalog_name}/{extension[1:]}/{filename}" def _publication_block_directory_path(block: Any, filename: str) -> str: @@ -86,4 +89,4 @@ def _publication_block_directory_path(block: Any, filename: str) -> str: """ publication_id = block.publication_id unique_name = f"{uuid.uuid4().hex}_{filename}" - return f"files/public/publications/{publication_id}/{unique_name}" + return f"publications/{publication_id}/{unique_name}" diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py index 9028f799..b3ae56a3 100644 --- a/api/utils/file_utils.py +++ b/api/utils/file_utils.py @@ -1,8 +1,9 @@ import mimetypes import os -from functools import lru_cache +import tempfile +from contextlib import contextmanager from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Iterator, Optional, Union import magic import pandas as pd @@ -16,18 +17,36 @@ def check_ext(file_object: Any) -> Optional[str]: return ext_type +@contextmanager +def local_copy(fieldfile: Any) -> Iterator[str]: + """Yield the path of a temporary local copy of a stored file. + + Media lives in S3, which has no local path, but pandas, libmagic and the + YAML/XML parsers need one. The copy keeps the file's extension (some + readers sniff it) and is removed on exit, even if the caller raises. + """ + suffix = os.path.splitext(fieldfile.name)[1] + with tempfile.NamedTemporaryFile(suffix=suffix) as tmp: + fieldfile.open("rb") + try: + for chunk in fieldfile.chunks(): + tmp.write(chunk) + finally: + fieldfile.close() + tmp.flush() + yield tmp.name + + def check_mime_type(file_object: Any) -> Any: # Prefer from_file: libmagic 5.46 misidentifies zip buffers as # application/octet-stream via from_buffer, while from_file works. - file_path = getattr(file_object, "path", None) - if file_path: - return magic.from_file(file_path, mime=True) + if hasattr(file_object, "storage"): + with local_copy(file_object) as file_path: + return magic.from_file(file_path, mime=True) return magic.from_buffer(file_object.read(), mime=True) -def file_validation( - file_object: Any, ext_object: Any, mapping: Dict[str, str] -) -> Optional[Any]: +def file_validation(file_object: Any, ext_object: Any, mapping: Dict[str, str]) -> Optional[Any]: ext_type = check_ext(ext_object) print("ext--", ext_type) mime_type = check_mime_type(file_object) @@ -78,7 +97,6 @@ def normalize_path(path: Union[str, Path]) -> str: return os.path.normpath(str(path)) -@lru_cache() def load_csv(filepath: str) -> pd.DataFrame: """Load CSV file into a pandas DataFrame. @@ -91,7 +109,6 @@ def load_csv(filepath: str) -> pd.DataFrame: return pd.read_csv(filepath) -@lru_cache() def load_excel(filepath: str) -> pd.DataFrame: """Load Excel file into a pandas DataFrame. @@ -104,7 +121,6 @@ def load_excel(filepath: str) -> pd.DataFrame: return pd.read_excel(filepath) -@lru_cache() def load_ods(filepath: str) -> pd.DataFrame: """Load ODS file into a pandas DataFrame. @@ -117,7 +133,6 @@ def load_ods(filepath: str) -> pd.DataFrame: return pd.read_excel(filepath, engine="odf") -@lru_cache() def load_parquet(filepath: str) -> pd.DataFrame: """Load Parquet file into a pandas DataFrame. @@ -130,7 +145,6 @@ def load_parquet(filepath: str) -> pd.DataFrame: return pd.read_parquet(filepath) -@lru_cache() def load_feather(filepath: str) -> pd.DataFrame: """Load Feather file into a pandas DataFrame. @@ -143,7 +157,6 @@ def load_feather(filepath: str) -> pd.DataFrame: return pd.read_feather(filepath) -@lru_cache() def load_json(filepath: str) -> pd.DataFrame: """Load JSON file into a pandas DataFrame. @@ -156,7 +169,6 @@ def load_json(filepath: str) -> pd.DataFrame: return pd.read_json(filepath) -@lru_cache() def load_tsv(filepath: str) -> pd.DataFrame: """Load TSV file into a pandas DataFrame. @@ -169,7 +181,6 @@ def load_tsv(filepath: str) -> pd.DataFrame: return pd.read_csv(filepath, sep="\t") -@lru_cache() def load_tabular_data(filepath: str, format: str) -> Optional[pd.DataFrame]: """Load tabular data from various file formats into a pandas DataFrame. diff --git a/api/utils/keycloak_utils.py b/api/utils/keycloak_utils.py index dca9e599..b088fbbb 100644 --- a/api/utils/keycloak_utils.py +++ b/api/utils/keycloak_utils.py @@ -150,7 +150,11 @@ def validate_token( # is not available to this client. try: user_info = self.keycloak_openid.userinfo(token) - return user_info + if isinstance(user_info, bytes): + import json + + user_info = json.loads(user_info.decode("utf-8")) + return user_info # type: ignore[return-value] except KeycloakError as userinfo_error: # If userinfo fails (e.g., 403), extract user info from token introspection logger.warning( diff --git a/api/utils/luma.py b/api/utils/luma.py new file mode 100644 index 00000000..d7415da2 --- /dev/null +++ b/api/utils/luma.py @@ -0,0 +1,175 @@ +""" +luma +──── +Parse a pasted Luma iframe into a small set of attributes. + +The stored value is not the raw HTML. ``src`` must be an ``https`` embed on +``lu.ma`` or ``luma.com``. Width, height, and the other presentation +attributes Luma includes are kept. Event handlers, ``srcdoc``, and any tag +other than one iframe are rejected. +""" + +import re +from html.parser import HTMLParser +from typing import Optional +from urllib.parse import urlparse, urlunparse + +from django.core.exceptions import ValidationError + +_LUMA_HOSTS = {"lu.ma", "www.lu.ma", "luma.com", "www.luma.com"} +_SIZE = re.compile(r"^\d{1,4}(px|%)?$") +_ALLOW = re.compile(r"^[A-Za-z0-9; _-]+$") +_UNSAFE_STYLE = re.compile( + r"javascript:|expression\s*\(|url\s*\(|@import|behavior\s*:|[<>\"']", + re.IGNORECASE, +) +_REFERRERS = { + "no-referrer", + "no-referrer-when-downgrade", + "origin", + "origin-when-cross-origin", + "same-origin", + "strict-origin", + "strict-origin-when-cross-origin", + "unsafe-url", +} +_LOADING = {"eager", "lazy"} + + +class _IframeParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.attrs: Optional[dict[str, str]] = None + self.iframe_count = 0 + self.other_tags = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + name = tag.lower() + if name == "iframe": + self.iframe_count += 1 + if self.iframe_count == 1: + self.attrs = {key.lower(): value or "" for key, value in attrs} + return + self.other_tags = True + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + self.handle_starttag(tag, attrs) + + +def canonical_luma_src(value: Optional[str]) -> Optional[str]: + """Return a canonical Luma embed URL, or None.""" + if not value or not value.strip(): + return None + parsed = urlparse(value.strip()) + if parsed.scheme != "https": + return None + if parsed.username or parsed.password: + return None + host = (parsed.hostname or "").lower() + if host not in _LUMA_HOSTS: + return None + if not parsed.path.startswith("/embed/"): + return None + return urlunparse(("https", host, parsed.path, "", parsed.query, "")) + + +def parse_luma_embed(value: Optional[str]) -> Optional[dict[str, object]]: + """Return whitelisted iframe attributes, or None when the paste is not a Luma embed.""" + if not value or not value.strip(): + return None + + trimmed = value.strip() + parser = _IframeParser() + parser.feed(trimmed) + parser.close() + if parser.other_tags or parser.iframe_count > 1: + return None + if parser.iframe_count == 1 and parser.attrs is not None: + return _attributes(parser.attrs) + if parser.iframe_count == 0: + src = canonical_luma_src(trimmed) + return {"src": src} if src else None + return None + + +def validate_luma_embed(value: Optional[str]) -> dict[str, object]: + """Return the whitelisted attributes, or raise ValidationError.""" + embed = parse_luma_embed(value) + if not embed: + raise ValidationError("Enter a valid Luma embed URL or iframe.") + return embed + + +def extract_luma_embed_url(value: Optional[str]) -> Optional[str]: + """Return only the embed URL. Kept for callers that do not need the other attributes.""" + embed = parse_luma_embed(value) + if not embed: + return None + src = embed.get("src") + return src if isinstance(src, str) else None + + +def _attributes(attrs: dict[str, str]) -> Optional[dict[str, object]]: + if any(name.startswith("on") or name in {"srcdoc", "formaction"} for name in attrs): + return None + src = canonical_luma_src(attrs.get("src")) + if not src: + return None + + embed: dict[str, object] = {"src": src} + width = _size(attrs.get("width")) + height = _size(attrs.get("height")) + style = _style(attrs.get("style")) + title = _plain(attrs.get("title"), 200) + allow = _allow(attrs.get("allow")) + frameborder = _size(attrs.get("frameborder")) + referrer = (attrs.get("referrerpolicy") or "").strip().lower() + loading = (attrs.get("loading") or "").strip().lower() + + if width: + embed["width"] = width + if height: + embed["height"] = height + if style: + embed["style"] = style + if title: + embed["title"] = title + if allow: + embed["allow"] = allow + if "allowfullscreen" in attrs: + embed["allowfullscreen"] = True + if frameborder: + embed["frameborder"] = frameborder + if referrer in _REFERRERS: + embed["referrerpolicy"] = referrer + if loading in _LOADING: + embed["loading"] = loading + return embed + + +def _size(value: Optional[str]) -> Optional[str]: + cleaned = (value or "").strip() + if _SIZE.match(cleaned): + return cleaned + return None + + +def _style(value: Optional[str]) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or len(cleaned) > 500 or _UNSAFE_STYLE.search(cleaned): + return None + return cleaned + + +def _plain(value: Optional[str], limit: int) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or "<" in cleaned or ">" in cleaned: + return None + return cleaned[:limit] + + +def _allow(value: Optional[str]) -> Optional[str]: + cleaned = " ".join((value or "").split()) + if not cleaned or len(cleaned) > 200 or not _ALLOW.match(cleaned): + return None + return cleaned diff --git a/api/utils/luma_cover.py b/api/utils/luma_cover.py new file mode 100644 index 00000000..7a72400d --- /dev/null +++ b/api/utils/luma_cover.py @@ -0,0 +1,95 @@ +""" +luma_cover +────────── +Look up the cover image of a Luma event from its stored embed URL. + +Used while building the event search document, so every index write +(publish, edit, ``search_index --populate`` on deploy) asks Luma afresh; no +value is carried over from a previous index. + +- The event id is taken from our own stored embed ``src`` + (``…/embed/event/evt-…``) and must match a strict pattern. +- The request always goes to ``https://api2.luma.com/event/get`` with that id; + redirects are not followed and at most ``_MAX_BYTES`` are read. +- Only an ``https`` image on ``images.lumacdn.com`` is accepted. + +Any failure (no embed, timeout, error status, unexpected body, no image) +returns None and is logged; indexing never fails because of Luma. +""" + +import json +import re +from typing import Any, Optional +from urllib.parse import urlparse + +import requests +import structlog + +logger = structlog.get_logger(__name__) + +_EVENT_ID = re.compile(r"^/embed/event/(evt-[A-Za-z0-9]{1,64})(?:/|$)") +_API_URL = "https://api2.luma.com/event/get" +_IMAGE_HOST = "images.lumacdn.com" +_MAX_BYTES = 1_000_000 +_TIMEOUT = 3 + + +def luma_event_id(src: Optional[str]) -> Optional[str]: + """Return the ``evt-…`` id from a Luma event embed URL, or None.""" + if not isinstance(src, str) or not src: + return None + match = _EVENT_ID.match(urlparse(src).path) + return match.group(1) if match else None + + +def cover_from_response(data: Any, event_id: str) -> Optional[str]: + """Return ``event.cover_url`` from an ``event/get`` response body, or None.""" + event = data.get("event") if isinstance(data, dict) else None + if not isinstance(event, dict) or event.get("api_id") != event_id: + return None + return _safe_image_url(event.get("cover_url")) + + +def fetch_luma_cover(embed_src: Optional[str]) -> Optional[str]: + """Return the cover image URL of the Luma event in ``embed_src``, or None.""" + event_id = luma_event_id(embed_src) + if not event_id: + return None + try: + with requests.get( + _API_URL, + params={"event_api_id": event_id}, + timeout=_TIMEOUT, + allow_redirects=False, + stream=True, + headers={"User-Agent": "DataSpace/1.0", "Accept": "application/json"}, + ) as response: + if response.status_code != 200: + logger.warning( + "luma_cover_fetch_failed", event_id=event_id, status=response.status_code + ) + return None + body = b"" + for chunk in response.iter_content(chunk_size=65536): + body += chunk + if len(body) > _MAX_BYTES: + logger.warning("luma_cover_response_too_large", event_id=event_id) + return None + cover = cover_from_response(json.loads(body), event_id) + except (requests.RequestException, ValueError) as exc: + logger.warning("luma_cover_fetch_failed", event_id=event_id, error=type(exc).__name__) + return None + if not cover: + logger.info("luma_cover_not_found", event_id=event_id) + return cover + + +def _safe_image_url(value: Any) -> Optional[str]: + if not isinstance(value, str) or not value or len(value) > 500: + return None + parsed = urlparse(value) + if parsed.scheme != "https" or parsed.hostname != _IMAGE_HOST: + return None + if parsed.username or parsed.password or parsed.port: + return None + return value diff --git a/api/utils/public_links.py b/api/utils/public_links.py new file mode 100644 index 00000000..03d05331 --- /dev/null +++ b/api/utils/public_links.py @@ -0,0 +1,118 @@ +"""Permanent public links: a short random ``public_id`` plus readable title words. + +A link looks like ``city-air-quality-index-k3x9m2qa7b1z``. Only the last part, +the ``public_id``, finds the record; the words come from the current title and +are there for people to read. Renaming a record changes the words, never the +id, so old links keep working. + +``public_id`` is unique per table, enforced by the database unique index. It is +never checked before saving (a check and an insert can race); a clash makes the +insert fail, and the save is retried with a new id. +""" + +from __future__ import annotations + +import re +import secrets +import string +import uuid +from typing import Callable, Optional + +from django.db import IntegrityError, models, transaction +from django.utils.text import slugify + +PUBLIC_ID_LENGTH = 12 +PUBLIC_ID_ALPHABET = string.ascii_lowercase + string.digits +URL_SLUG_WORDS_MAX = 50 + +_PUBLIC_ID_RE = re.compile(rf"[a-z0-9]{{{PUBLIC_ID_LENGTH}}}") + + +def generate_public_id() -> str: + """12 random characters from a-z and 0-9 (36^12, about 4.7 x 10^18 values).""" + return "".join(secrets.choice(PUBLIC_ID_ALPHABET) for _ in range(PUBLIC_ID_LENGTH)) + + +def save_with_public_id( + instance: models.Model, + save_fn: Callable[[], None], + field: str = "public_id", + max_attempts: int = 5, +) -> None: + """Save ``instance``, giving it a new ``public_id`` first if it has none. + + If the insert fails because another row already has that id, a new id is + generated and the save runs again. Any other database error is raised + unchanged. Each attempt runs in its own savepoint, so a failed attempt does + not break the caller's transaction. + """ + if getattr(instance, field): + save_fn() + return + + for attempt in range(1, max_attempts + 1): + setattr(instance, field, generate_public_id()) + try: + with transaction.atomic(): + save_fn() + return + except IntegrityError as exc: + if field not in str(exc) or attempt == max_attempts: + raise + + +def build_url_slug(title: Optional[str], public_id: str) -> str: + """Link part for a record: title words (max 50 chars, cut between words) + id. + + Returns only the id when the title has no Latin letters or digits (for + example a title written only in Hindi), since those are dropped by slugify. + """ + words = slugify(title or "") + if len(words) > URL_SLUG_WORDS_MAX: + cut = words[: URL_SLUG_WORDS_MAX + 1].rsplit("-", 1)[0] + # A single word longer than the limit has no boundary to cut at. + words = cut if len(cut) <= URL_SLUG_WORDS_MAX else words[:URL_SLUG_WORDS_MAX] + words = words.strip("-") + return f"{words}-{public_id}" if words else public_id + + +def parse_public_id(link_part: Optional[str]) -> Optional[str]: + """The ``public_id`` at the end of a link part, or None if there is none. + + ``city-air-quality-index-k3x9m2qa7b1z`` and ``k3x9m2qa7b1z`` both give + ``k3x9m2qa7b1z``. A UUID or a number gives None. An old slug whose last word + happens to be 12 letters long does parse, so callers should fall back to + their old lookup both when this returns None and when no record has the id. + """ + if not link_part: + return None + link_part = link_part.strip().strip("/") + # A UUID's last group is also 12 hex characters; it is an old link, not an id. + try: + uuid.UUID(link_part) + return None + except ValueError: + pass + candidate = link_part.rsplit("-", 1)[-1].lower() + return candidate if _PUBLIC_ID_RE.fullmatch(candidate) else None + + +def fill_missing_public_ids(model: type[models.Model], field: str = "public_id") -> int: + """Give every row of ``model`` that has no ``public_id`` a new one. + + For data migrations that add the column to an existing table, run before the + unique index is added. Uniqueness is kept here with a set of ids already in + the table, since the index that would catch a clash does not exist yet. + Returns the number of rows filled. + """ + manager = model._default_manager # type: ignore[attr-defined] + used = set(manager.exclude(**{f"{field}__isnull": True}).values_list(field, flat=True)) + filled = 0 + for pk in list(manager.filter(**{f"{field}__isnull": True}).values_list("pk", flat=True)): + value = generate_public_id() + while value in used: + value = generate_public_id() + used.add(value) + manager.filter(pk=pk).update(**{field: value}) + filled += 1 + return filled diff --git a/api/utils/qs_utils.py b/api/utils/qs_utils.py index 5e6fc2c6..803384ad 100644 --- a/api/utils/qs_utils.py +++ b/api/utils/qs_utils.py @@ -8,10 +8,10 @@ from __future__ import annotations -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence, Type import strawberry -from django.db.models import Q, QuerySet +from django.db.models import Count, Q, QuerySet MAX_LIMIT = 100 @@ -50,6 +50,12 @@ class SortSpec: direction: str = "asc" +@strawberry.type(description="Rows per status, for a table's tab labels.") +class StatusCount: + status: str + count: int + + def _check_allowed(kind: str, field: str, allowed: Optional[Sequence[str]]) -> None: if allowed is not None and field not in allowed: raise ValueError( @@ -128,3 +134,23 @@ def get_pagination_window( if sort_options: queryset = apply_sorting(queryset, sort_options, sorting_allowed_fields) return apply_pagination(queryset, limit, offset) + + +def status_counts( + queryset: QuerySet, + filters: Optional[Sequence[FilterSpec]] = None, + allowed_fields: Optional[Sequence[str]] = None, + status_field: str = "status", + item_type: Type[Any] = StatusCount, +) -> List[Any]: + """Count rows per status under every filter except the status one, so tab + labels stay the same whichever tab is open. ``item_type`` lets a module keep + its own GraphQL type name (``DatasetStatusCount``).""" + others = [f for f in (filters or []) if (f.field or "").strip() != status_field] + rows = ( + apply_filters(queryset, others, allowed_fields) + .order_by() + .values(status_field) + .annotate(n=Count("pk", distinct=True)) + ) + return [item_type(status=row[status_field], count=row["n"]) for row in rows] diff --git a/api/utils/storage.py b/api/utils/storage.py new file mode 100644 index 00000000..7cdb679e --- /dev/null +++ b/api/utils/storage.py @@ -0,0 +1,35 @@ +"""Media storages for model file fields. + +Every ``FileField``/``ImageField`` names one of these through ``storage=``. +They are callables, not instances, so migrations record a function reference +instead of bucket settings, and tests can swap the backend through STORAGES. +""" + +from django.core.files.storage import Storage, storages + +# Folder prefixes from the on-disk era, when MEDIA_ROOT was files/public and the +# upload_to helpers added their own "files/..." on top. The S3 storages already +# put every key under /public or /private, so these only repeat it. +_LEGACY_PREFIXES = ("files/public/", "files/") + + +def public_storage() -> Storage: + """Storage for files anyone may read: unsigned, cacheable URLs.""" + return storages["public"] + + +def private_storage() -> Storage: + """Storage for gated files: short-lived presigned URLs only.""" + return storages["private"] + + +def normalize_media_name(name: str) -> str: + """Strip the legacy ``files/public/`` or ``files/`` prefix from a stored name. + + ``files/public/organizations/acme/png/logo.png`` -> ``organizations/acme/png/logo.png``. + Names without a legacy prefix are returned unchanged. + """ + for prefix in _LEGACY_PREFIXES: + if name.startswith(prefix): + return name[len(prefix) :] + return name diff --git a/api/utils/version_detection.py b/api/utils/version_detection.py deleted file mode 100644 index 809e9db6..00000000 --- a/api/utils/version_detection.py +++ /dev/null @@ -1,327 +0,0 @@ -import json -import os -import xml.etree.ElementTree as ET -from typing import Any, Dict, List, Set, Tuple, Union - -import pandas as pd -import structlog -from deepdiff import DeepDiff # type: ignore - -logger = structlog.getLogger(__name__) - - -def detect_version_change_type(old_file_path: str, new_file_path: str) -> str: - """ - Analyze changes between two versions of a file and determine the appropriate - version increment type (major, minor, or patch). - - Args: - old_file_path: Path to the previous version of the file - new_file_path: Path to the new version of the file - - Returns: - String indicating the type of version change: "major", "minor", or "patch" - """ - # Get file extensions - _, old_ext = os.path.splitext(old_file_path) - _, new_ext = os.path.splitext(new_file_path) - - # If file format changed, it's a major change - if old_ext.lower() != new_ext.lower(): - logger.info( - f"File format changed from {old_ext} to {new_ext} - major version change" - ) - return "major" - - # Handle different file types - if old_ext.lower() in [".csv", ".tsv"]: - return _detect_tabular_changes(old_file_path, new_file_path) - elif old_ext.lower() in [".json"]: - return _detect_json_changes(old_file_path, new_file_path) - elif old_ext.lower() in [".xml"]: - return _detect_xml_changes(old_file_path, new_file_path) - else: - # For other file types, default to comparing file size - return _detect_generic_changes(old_file_path, new_file_path) - - -def _detect_tabular_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in tabular data files (CSV, TSV) using pandas - """ - try: - # Read files with pandas - old_df = pd.read_csv(old_file_path, encoding="utf-8") - new_df = pd.read_csv(new_file_path, encoding="utf-8") - - # Check for schema changes - old_columns = set(old_df.columns) - new_columns = set(new_df.columns) - - # If columns were added or removed, it's a major change - if old_columns != new_columns: - added = new_columns - old_columns - removed = old_columns - new_columns - logger.info( - f"Schema changed - added columns: {added}, removed columns: {removed} - major version change" - ) - return "major" - - # Check for data type changes in columns - for col in old_columns: - if old_df[col].dtype != new_df[col].dtype: - logger.info( - f"Data type changed for column {col} - major version change" - ) - return "major" - - # Check for significant row count changes - if len(old_df) != len(new_df): - # Row count changed by more than 10% - change_percentage = ( - abs(len(new_df) - len(old_df)) / max(len(old_df), 1) * 100 - ) - if change_percentage > 10: - logger.info( - f"Row count changed by {change_percentage:.2f}% - minor version change" - ) - return "minor" - - # Use pandas built-in comparison for data changes - # Limit to common rows to avoid index errors - min_rows = min(len(old_df), len(new_df)) - if min_rows > 0: - # Truncate both dataframes to the same length - old_sample = old_df.iloc[:min_rows].reset_index(drop=True) - new_sample = new_df.iloc[:min_rows].reset_index(drop=True) - - # Use pandas compare function (available in pandas >= 1.1.0) - try: - # Get differences between dataframes - diff_df = new_sample.compare(old_sample) - - # Calculate change percentage - total_cells = min_rows * len(old_columns) - changed_cells = diff_df.size # Number of changed cells - - change_percentage = ( - (changed_cells / total_cells) * 100 if total_cells > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"Data changed significantly ({change_percentage:.2f}% of cells) - minor version change" - ) - return "minor" - elif change_percentage > 0: - logger.info( - f"Small data changes ({change_percentage:.2f}% of cells) - patch version change" - ) - return "patch" - except Exception as compare_error: - # Fallback if compare fails (older pandas version or other issues) - logger.warning( - f"Pandas compare failed: {str(compare_error)}, using manual comparison" - ) - - # Manual comparison as fallback - changes = 0 - total_cells = min_rows * len(old_columns) - - for col in old_columns: - # Compare column values - unequal_mask = old_sample[col].values != new_sample[col].values - changes += unequal_mask.sum() # type: ignore - - change_percentage = ( - (changes / total_cells) * 100 if total_cells > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"Data changed significantly ({change_percentage:.2f}% of cells) - minor version change" - ) - return "minor" - elif change_percentage > 0: - logger.info( - f"Small data changes ({change_percentage:.2f}% of cells) - patch version change" - ) - return "patch" - - # Default to patch for minimal changes - return "patch" - - except Exception as e: - logger.error(f"Error analyzing tabular files: {str(e)}") - # Default to minor if we can't analyze properly - return "minor" - - -def _detect_json_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in JSON files using DeepDiff - """ - try: - with open(old_file_path, "r", encoding="utf-8") as f: - old_data = json.load(f) - - with open(new_file_path, "r", encoding="utf-8") as f: - new_data = json.load(f) - - # Use DeepDiff for efficient and accurate comparison - diff = DeepDiff(old_data, new_data, ignore_order=True) - - # Check for structure changes (major) - if diff.get("dictionary_item_added") or diff.get("dictionary_item_removed"): - logger.info(f"JSON structure changed - major version change") - return "major" - - # Check for type changes (major) - if diff.get("type_changes"): - logger.info(f"JSON data types changed - major version change") - return "major" - - # Check for significant value changes (minor) - value_changes = len(diff.get("values_changed", {})) - if isinstance(old_data, dict) and isinstance(new_data, dict): - total_values = _count_values(old_data) - change_percentage = ( - (value_changes / total_values) * 100 if total_values > 0 else 0 - ) - - if change_percentage > 30: - logger.info( - f"JSON values changed significantly ({change_percentage:.2f}%) - minor version change" - ) - return "minor" - - # Check for array changes (minor) - if diff.get("iterable_item_added") or diff.get("iterable_item_removed"): - logger.info(f"JSON array items changed - minor version change") - return "minor" - - # If there are any changes at all, it's at least a patch - if diff: - return "patch" - - # No changes detected - return "patch" - - except Exception as e: - logger.error(f"Error analyzing JSON files: {str(e)}") - return "minor" - - -def _detect_xml_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in XML files - """ - try: - import xml.etree.ElementTree as ET - - old_tree = ET.parse(old_file_path) - new_tree = ET.parse(new_file_path) - - old_root = old_tree.getroot() - new_root = new_tree.getroot() - - # Check for tag changes - old_tags = _get_xml_tags(old_root) - new_tags = _get_xml_tags(new_root) - - if old_tags != new_tags: - logger.info(f"XML structure changed - major version change") - return "major" - - # Check for attribute changes - old_attrs = _get_xml_attributes(old_root) - new_attrs = _get_xml_attributes(new_root) - - if old_attrs != new_attrs: - logger.info(f"XML attributes changed - minor version change") - return "minor" - - # Default to patch for content changes - return "patch" - - except Exception as e: - logger.error(f"Error analyzing XML files: {str(e)}") - return "minor" - - -def _detect_generic_changes(old_file_path: str, new_file_path: str) -> str: - """ - Detect changes in generic files based on size - """ - old_size = os.path.getsize(old_file_path) - new_size = os.path.getsize(new_file_path) - - # Calculate size change percentage - size_change_percentage = abs(new_size - old_size) / max(old_size, 1) * 100 - - if size_change_percentage > 50: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - major version change" - ) - return "major" - elif size_change_percentage > 10: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - minor version change" - ) - return "minor" - else: - logger.info( - f"File size changed by {size_change_percentage:.2f}% - patch version change" - ) - return "patch" - - -def _flatten_dict_keys(d: Dict, parent_key: str = "") -> List[str]: - """ - Flatten nested dictionary keys - """ - items = [] - for k, v in d.items(): - new_key = f"{parent_key}.{k}" if parent_key else k - if isinstance(v, dict): - items.extend(_flatten_dict_keys(v, new_key)) - else: - items.append(new_key) - return items - - -def _count_values(obj: Union[Dict, List, Any], count: int = 0) -> int: - """ - Count the total number of values in a nested structure - """ - if isinstance(obj, dict): - for k, v in obj.items(): - count += 1 # Count the key-value pair itself - if isinstance(v, (dict, list)): - count = _count_values(v, count) - elif isinstance(obj, list): - count += len(obj) # Count the list items - for item in obj: - if isinstance(item, (dict, list)): - count = _count_values(item, count) - return count - - -def _get_xml_tags(element: ET.Element) -> Set[str]: - """ - Get all unique tag names in an XML tree - """ - tags = {element.tag} - for child in element: - tags.update(_get_xml_tags(child)) - return tags - - -def _get_xml_attributes(element: ET.Element) -> Set[Tuple[str, str]]: - """ - Get all attributes in an XML tree as (element_tag, attribute_name) pairs - """ - attrs = {(element.tag, attr) for attr in element.attrib.keys()} - for child in element: - attrs.update(_get_xml_attributes(child)) - return attrs diff --git a/api/views/dataset_data.py b/api/views/dataset_data.py new file mode 100644 index 00000000..f6f0b6a7 --- /dev/null +++ b/api/views/dataset_data.py @@ -0,0 +1,359 @@ +"""HTTP endpoints for fetching indexed dataset/resource data from data_db. + +Endpoints: + +- ``GET /api/resources//data/`` — fetch indexed data for a single + resource with column-based filtering. +- ``GET /api/datasets//data/`` — fetch indexed data for a dataset. + By default operates on the dataset's first indexed resource. Pass + ``?resource_id=`` to target a specific resource. +- ``GET /api/datasets//prompts/`` — fetch indexed data for a + PromptDataset, restricted to ``dataset_type=PROMPT`` and exposing extra + prompt-specific filter shorthands. + +All endpoints accept these query params: + +- ``columns`` — comma-separated list of columns to project. +- ``limit`` (default 100, max 10000), ``offset`` (default 0). +- ``order_by`` — comma-separated columns; prefix with ``-`` for DESC. +- ``count`` — ``true``/``false`` (default ``true``) to include total row count. +- Any other query param is interpreted as a data-column filter, optionally + with operator suffix, e.g. ``?price__gte=10&category=books``. Repeated keys + produce a list (used naturally for ``__in``/``__nin``). +""" + +import uuid +from typing import Any, Dict, List, Optional, Tuple + +import structlog +from django.http import HttpRequest +from rest_framework.permissions import AllowAny +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.models import Dataset, Resource, ResourceDataTable +from api.models.PromptDataset import PromptDataset +from api.utils.data_indexing import ( + DEFAULT_FETCH_LIMIT, + MAX_FETCH_LIMIT, + DataFetchError, + fetch_resource_data, + get_resource_columns, +) +from api.utils.enums import DatasetStatus, DatasetType + +logger = structlog.get_logger(__name__) + +# Reserved query parameters that are NOT treated as column filters. +_RESERVED_PARAMS = { + "columns", + "limit", + "offset", + "order_by", + "count", + "resource_id", + "format", +} + + +def _parse_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "y", "on"} + + +def _parse_int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _parse_csv(value: Optional[str]) -> Optional[List[str]]: + if not value: + return None + parts = [p.strip() for p in value.split(",") if p.strip()] + return parts or None + + +def _extract_filters(query_params: Any, reserved: Optional[set] = None) -> Dict[str, Any]: + """Pull non-reserved query params as filter dict. + + Repeated keys collapse into lists so callers can use + ``?col__in=a&col__in=b``. ``__in``/``__nin`` always produce a list, even + for a single value. + """ + reserved_set = reserved if reserved is not None else _RESERVED_PARAMS + filters: Dict[str, Any] = {} + # query_params is a QueryDict; use .lists() if available + if hasattr(query_params, "lists"): + items = query_params.lists() + else: + items = [(k, [v]) for k, v in query_params.items()] + + for key, values in items: + if key in reserved_set: + continue + if not values: + continue + op_suffix = key.rsplit("__", 1)[-1] if "__" in key else None + if op_suffix in ("in", "nin"): + # Allow comma-separated single value too + collected: List[Any] = [] + for v in values: + if isinstance(v, str) and "," in v: + collected.extend([p for p in (s.strip() for s in v.split(",")) if p]) + else: + collected.append(v) + filters[key] = collected + else: + # Last value wins for non-list operators + filters[key] = values[-1] + return filters + + +def _user_can_access_dataset(request: HttpRequest, dataset: Dataset) -> bool: + """Allow access to PUBLISHED datasets, otherwise require owner/org-member.""" + if dataset.status == DatasetStatus.PUBLISHED.value: + return True + user = getattr(request, "user", None) + if not user or not user.is_authenticated: + return False + if user.is_superuser: + return True + if dataset.user_id and dataset.user_id == user.id: + return True + if dataset.organization_id: + # Lazy import to avoid circular imports at module load + from authorization.models import OrganizationMembership + + return OrganizationMembership.objects.filter( + user=user, organization_id=dataset.organization_id + ).exists() + return False + + +def _resolve_dataset_resource( + dataset: Dataset, resource_id: Optional[str] +) -> Tuple[Optional[Resource], Optional[Response]]: + """Pick a Resource for a dataset-level data fetch. + + Returns ``(resource, error_response)`` — exactly one is non-None. + """ + if resource_id: + try: + resource = dataset.resources.get(id=resource_id) + except Resource.DoesNotExist: + return None, Response( + {"error": f"Resource {resource_id} not found in dataset {dataset.id}"}, + status=404, + ) + return resource, None + + # Default: first resource that has indexed data + indexed_table = ( + ResourceDataTable.objects.filter(resource__dataset=dataset).order_by("created").first() + ) + if indexed_table is None: + return None, Response( + { + "error": ( + "Dataset has no indexed (tabular) resources. " + "Pass ?resource_id= or upload a CSV/XLSX/Parquet/JSON file." + ) + }, + status=404, + ) + return indexed_table.resource, None + + +def _fetch_and_respond( + request: Request, + resource: Resource, + extra_filters: Optional[Dict[str, Any]] = None, + reserved: Optional[set] = None, + extra_response: Optional[Dict[str, Any]] = None, +) -> Response: + """Common path: parse query params, run fetch_resource_data, return JSON.""" + qp = request.query_params # type: ignore[attr-defined] + + columns = _parse_csv(qp.get("columns")) + order_by = _parse_csv(qp.get("order_by")) + limit = _parse_int(qp.get("limit"), DEFAULT_FETCH_LIMIT) + offset = _parse_int(qp.get("offset"), 0) + count = _parse_bool(qp.get("count"), default=True) + + filters = _extract_filters(qp, reserved=reserved) + if extra_filters: + filters.update(extra_filters) + + try: + result = fetch_resource_data( + resource=resource, + filters=filters, + columns=columns, + limit=limit, + offset=offset, + order_by=order_by, + count=count, + ) + except DataFetchError as e: + return Response({"error": str(e)}, status=400) + except Exception as e: # pragma: no cover — defensive + logger.exception( + "fetch_resource_data failed", + resource_id=str(resource.id), + error=str(e), + ) + return Response({"error": "Failed to fetch data"}, status=500) + + available = get_resource_columns(resource) + + payload: Dict[str, Any] = { + "resource_id": str(resource.id), + "dataset_id": str(resource.dataset_id), + "available_columns": available, + "max_limit": MAX_FETCH_LIMIT, + **result, + } + if extra_response: + payload.update(extra_response) + return Response(payload) + + +class ResourceDataView(APIView): + """Return indexed data for a specific resource.""" + + permission_classes = [AllowAny] + + def get(self, request: Request, resource_id: uuid.UUID) -> Response: + try: + resource = Resource.objects.select_related("dataset").get(id=resource_id) + except Resource.DoesNotExist: + return Response({"error": "Resource not found"}, status=404) + + if not _user_can_access_dataset(request, resource.dataset): # type: ignore[attr-defined] + return Response({"error": "Not authorized"}, status=403) + + return _fetch_and_respond(request, resource) + + +class DatasetDataView(APIView): + """Return indexed data for a dataset (one resource at a time).""" + + permission_classes = [AllowAny] + + def get(self, request: Request, dataset_id: uuid.UUID) -> Response: + try: + dataset = Dataset.objects.get(id=dataset_id) + except Dataset.DoesNotExist: + return Response({"error": "Dataset not found"}, status=404) + + if not _user_can_access_dataset(request, dataset): + return Response({"error": "Not authorized"}, status=403) + + resource_id = request.query_params.get("resource_id") # type: ignore[attr-defined] + resource, err = _resolve_dataset_resource(dataset, resource_id) + if err is not None: + return err + assert resource is not None + return _fetch_and_respond(request, resource) + + +class PromptDatasetDataView(APIView): + """Return indexed data for a PromptDataset. + + Same query semantics as :class:`DatasetDataView`, but the dataset must be + of type ``PROMPT``. Convenience query params (translated to column + filters when those columns exist on the data): + + - ``prompt_contains`` -> ``prompt__icontains`` + - ``response_contains`` -> ``response__icontains`` (or ``completion``) + - ``min_length``/``max_length`` -> ``length__gte``/``length__lte`` + """ + + permission_classes = [AllowAny] + + # Conventional column names we look for on prompt data tables. + _PROMPT_COL_CANDIDATES = ("prompt", "input", "instruction", "question") + _RESPONSE_COL_CANDIDATES = ("response", "completion", "answer", "output") + _LENGTH_COL_CANDIDATES = ("length", "prompt_length", "tokens", "token_count") + + def _first_present(self, available: List[str], candidates: Tuple[str, ...]) -> Optional[str]: + lower_map = {c.lower(): c for c in available} + for cand in candidates: + if cand in lower_map: + return lower_map[cand] + return None + + def get(self, request: Request, dataset_id: uuid.UUID) -> Response: + try: + prompt_dataset = PromptDataset.objects.get(dataset_ptr_id=dataset_id) + except PromptDataset.DoesNotExist: + return Response( + {"error": f"Dataset {dataset_id} is not a prompt dataset"}, + status=404, + ) + + if prompt_dataset.dataset_type != DatasetType.PROMPT.value: + return Response( + {"error": f"Dataset {dataset_id} is not a prompt dataset"}, + status=400, + ) + + if not _user_can_access_dataset(request, prompt_dataset): + return Response({"error": "Not authorized"}, status=403) + + resource_id = request.query_params.get("resource_id") # type: ignore[attr-defined] + resource, err = _resolve_dataset_resource(prompt_dataset, resource_id) + if err is not None: + return err + assert resource is not None + + # Map prompt-specific shorthands to underlying column filters + available = get_resource_columns(resource) + qp = request.query_params # type: ignore[attr-defined] + extra: Dict[str, Any] = {} + + prompt_col = self._first_present(available, self._PROMPT_COL_CANDIDATES) + response_col = self._first_present(available, self._RESPONSE_COL_CANDIDATES) + length_col = self._first_present(available, self._LENGTH_COL_CANDIDATES) + + prompt_q = qp.get("prompt_contains") + if prompt_q and prompt_col: + extra[f"{prompt_col}__icontains"] = prompt_q + + response_q = qp.get("response_contains") + if response_q and response_col: + extra[f"{response_col}__icontains"] = response_q + + min_len = qp.get("min_length") + if min_len and length_col: + extra[f"{length_col}__gte"] = min_len + + max_len = qp.get("max_length") + if max_len and length_col: + extra[f"{length_col}__lte"] = max_len + + local_reserved = _RESERVED_PARAMS | { + "prompt_contains", + "response_contains", + "min_length", + "max_length", + } + + return _fetch_and_respond( + request, + resource, + extra_filters=extra, + reserved=local_reserved, + extra_response={ + "dataset_type": prompt_dataset.dataset_type, + "prompt_column": prompt_col, + "response_column": response_col, + "length_column": length_col, + }, + ) diff --git a/api/views/download_view.py b/api/views/download_view.py index 5d106483..d6668bae 100644 --- a/api/views/download_view.py +++ b/api/views/download_view.py @@ -6,7 +6,14 @@ from asgiref.sync import sync_to_async from django.core.exceptions import ObjectDoesNotExist from django.core.files.uploadedfile import UploadedFile -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse +from django.http import ( + FileResponse, + HttpRequest, + HttpResponse, + HttpResponseRedirect, + JsonResponse, +) +from django.http.response import HttpResponseBase from pyecharts.charts.chart import Chart from pyecharts.render import make_snapshot from selenium import webdriver @@ -46,7 +53,7 @@ def get_chart_image(id: uuid.UUID) -> ResourceChartImage: @sync_to_async def get_resource_response( resource: Resource, request: Optional[HttpRequest] = None -) -> HttpResponse: +) -> HttpResponseBase: """Get file response for a resource.""" # Link-only (platform-imported) resources: we hold no bytes, send the # user to the file on the source platform. Still counts as a download. @@ -62,7 +69,8 @@ def get_resource_response( if not file_details or not file_details.file: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse(file_details.file.read(), content_type="application/octet-stream") + # Stream from storage instead of reading the whole object into memory. + response = FileResponse(file_details.file.open("rb"), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"resource_{resource.name}.csv" @@ -98,12 +106,12 @@ def _track_download(resource: Resource, request: Optional[HttpRequest]) -> None: @sync_to_async -def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: +def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponseBase: """Get file response for a chart image.""" if not chart_image.image: return JsonResponse({"error": "File not found"}, status=404) - response = HttpResponse(chart_image.image.read(), content_type="application/octet-stream") + response = FileResponse(chart_image.image.open("rb"), content_type="application/octet-stream") # Handle filename and basename explicitly default_name = f"chart_{chart_image.id}.png" @@ -119,7 +127,7 @@ def get_chart_image_response(chart_image: ResourceChartImage) -> HttpResponse: return response -async def download(request: HttpRequest, type: str, id: uuid.UUID) -> HttpResponse: +async def download(request: HttpRequest, type: str, id: uuid.UUID) -> HttpResponseBase: """Handle download requests for resources, chart images, and charts.""" try: if type == "resource": diff --git a/api/views/legacy_media.py b/api/views/legacy_media.py new file mode 100644 index 00000000..e8900a24 --- /dev/null +++ b/api/views/legacy_media.py @@ -0,0 +1,36 @@ +""" +legacy_media +──────────── +Keeps links issued before the move to S3 working. Media used to be served from +disk at ``/public/``; that path now redirects to the file's S3 URL. + +Names are normalised first: rows created on disk carried a ``files/public/`` or +``files/`` prefix that S3 keys no longer have, so ``/public/files/public/x.png`` +redirects to the file now stored as ``x.png``. + +Only public files redirect. The old route also exposed resources and draft +files whenever DEBUG was on; anything not in public storage is a 404 here, so +this route can never hand out a private object. +""" + +import posixpath + +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse + +from api.utils.storage import normalize_media_name, public_storage + + +def legacy_media_redirect(request: HttpRequest, name: str) -> HttpResponse: + """Redirect an old ``/public/`` link to the public file's S3 URL, else 404.""" + normalised = posixpath.normpath(name) + if normalised != name or normalised.startswith(("/", "..")): + return JsonResponse({"error": "Not found"}, status=404) + + storage = public_storage() + current = normalize_media_name(name) + if not storage.exists(current): + return JsonResponse({"error": "Not found"}, status=404) + + response = HttpResponseRedirect(storage.url(current)) + response["Cache-Control"] = "public, max-age=86400" + return response diff --git a/api/views/metadata_export.py b/api/views/metadata_export.py index a3df43cb..bb7bf5c3 100644 --- a/api/views/metadata_export.py +++ b/api/views/metadata_export.py @@ -43,11 +43,18 @@ def metadata_export(request: HttpRequest, dataset_id: uuid.UUID) -> HttpResponse standard = (request.GET.get("standard") or "dcat").strip().lower() fmt = (request.GET.get("format") or "jsonld").strip().lower() + # Only these two inputs are the caller's fault. Fixed messages: never echo + # an exception to a public client. + options = export_options() + if standard not in options: + return JsonResponse({"error": "Unknown standard", "options": options}, status=400) + if fmt not in options[standard]["formats"]: + return JsonResponse( + {"error": "Format not available for this standard", "options": options}, status=400 + ) try: body, content_type, ext, report = export_dataset(dataset, standard, fmt) - except ValueError as exc: - return JsonResponse({"error": str(exc), "options": export_options()}, status=400) - except Exception as exc: # pragma: no cover - defensive; never 500 on a public page + except Exception as exc: # internal failure: log it, return a fixed message logger.error( "metadata_export_failed", dataset_id=str(dataset_id), standard=standard, error=str(exc) ) diff --git a/api/views/paginated_elastic_view.py b/api/views/paginated_elastic_view.py index 9f81f09a..0277a071 100644 --- a/api/views/paginated_elastic_view.py +++ b/api/views/paginated_elastic_view.py @@ -186,6 +186,20 @@ def get(self, request: HttpRequest) -> Response: for agg in running_status_agg: aggregations["running_status"][agg["key"]] = agg["doc_count"] + if "location" in aggregations: + location_agg = aggregations["location"]["buckets"] + aggregations.pop("location") + aggregations["location"] = {} + for agg in location_agg: + aggregations["location"][agg["key"]] = agg["doc_count"] + + if "is_individual_event" in aggregations: + is_individual_event_agg = aggregations["is_individual_event"]["buckets"] + aggregations.pop("is_individual_event") + aggregations["is_individual_event"] = {} + for agg in is_individual_event_agg: + aggregations["is_individual_event"][agg["key_as_string"]] = agg["doc_count"] + if "is_individual_usecase" in aggregations: is_individual_usecase_agg = aggregations["is_individual_usecase"]["buckets"] aggregations.pop("is_individual_usecase") diff --git a/api/views/publication_download_view.py b/api/views/publication_download_view.py index deac2435..8f076909 100644 --- a/api/views/publication_download_view.py +++ b/api/views/publication_download_view.py @@ -15,7 +15,8 @@ from typing import Any from django.db.models import F -from django.http import HttpRequest, HttpResponse, JsonResponse +from django.http import FileResponse, HttpRequest, JsonResponse +from django.http.response import HttpResponseBase from api.models import Publication, PublicationBlock from api.services.publication_blocks import can_access_block_file @@ -25,7 +26,7 @@ _INLINE_CONTENT_TYPES = {".pdf": "application/pdf"} -def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponse: +def publication_block_download(request: HttpRequest, block_id: Any) -> HttpResponseBase: """Serve a block's file if the caller may see its resource, else 404.""" # Find the block and its parent resource. try: @@ -52,7 +53,7 @@ def publication_block_download(request: HttpRequest, block_id: Any) -> HttpRespo return _build_file_response(block) -def _build_file_response(block: PublicationBlock) -> HttpResponse: +def _build_file_response(block: PublicationBlock) -> FileResponse: """Build the HTTP file response with the right content type and disposition.""" stored_name = block.file.name or "" extension = os.path.splitext(stored_name)[1].lower() @@ -60,6 +61,6 @@ def _build_file_response(block: PublicationBlock) -> HttpResponse: disposition = "inline" if extension in _INLINE_CONTENT_TYPES else "attachment" filename = block.file_name or os.path.basename(stored_name) - response = HttpResponse(block.file.read(), content_type=content_type) + response = FileResponse(block.file.open("rb"), content_type=content_type) response["Content-Disposition"] = f'{disposition}; filename="{filename}"' return response diff --git a/api/views/search_aimodel.py b/api/views/search_aimodel.py index dbf5ce6c..bb5a193c 100644 --- a/api/views/search_aimodel.py +++ b/api/views/search_aimodel.py @@ -20,6 +20,9 @@ class AIModelDocumentSerializer(serializers.ModelSerializer): """Serializer for AIModel document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + tags = serializers.ListField() sectors = serializers.ListField() geographies = serializers.ListField() @@ -77,6 +80,8 @@ class Meta: model = AIModel fields = [ "id", + "public_id", + "url_slug", "name", "display_name", "description", diff --git a/api/views/search_collaborative.py b/api/views/search_collaborative.py index 1bb7120c..a17a2344 100644 --- a/api/views/search_collaborative.py +++ b/api/views/search_collaborative.py @@ -55,6 +55,8 @@ def to_internal_value(self, data: Dict[str, Any]) -> Dict[str, Any]: class CollaborativeDocumentSerializer(serializers.ModelSerializer): + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) metadata = CollaborativeMetadataSerializer(many=True) tags = serializers.ListField() sectors = serializers.ListField(default=[]) @@ -107,6 +109,8 @@ class Meta: model = Collaborative fields = [ "id", + "public_id", + "url_slug", "title", "summary", "slug", diff --git a/api/views/search_dataset.py b/api/views/search_dataset.py index 5821803b..36ad6e7a 100644 --- a/api/views/search_dataset.py +++ b/api/views/search_dataset.py @@ -85,6 +85,8 @@ class DatasetDocumentSerializer(serializers.ModelSerializer): geographies = serializers.ListField() has_charts = serializers.BooleanField() slug = serializers.CharField() + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) download_count = serializers.IntegerField() trending_score = serializers.FloatField(required=False) is_individual_dataset = serializers.BooleanField() @@ -110,6 +112,8 @@ class Meta: "title", "description", "slug", + "public_id", + "url_slug", "created", "modified", "status", diff --git a/api/views/search_event.py b/api/views/search_event.py new file mode 100644 index 00000000..40c54757 --- /dev/null +++ b/api/views/search_event.py @@ -0,0 +1,169 @@ +"""Search view for Event using Elasticsearch. + +Only published events are in the index (the document's ``should_index_object`` +gate), so this public endpoint never leaks drafts. Filters: location, owner +type (``is_individual_event``) and ``timeframe`` (upcoming / past). +""" + +from typing import Any, Dict, List, Optional, Tuple, Union + +import structlog +from elasticsearch_dsl import Q as ESQ +from elasticsearch_dsl import Search +from elasticsearch_dsl.query import Query as ESQuery +from rest_framework import serializers +from rest_framework.permissions import AllowAny + +from api.models.Event import Event +from api.utils.search_query import nested_paths, text_query +from api.utils.telemetry_utils import trace_method +from api.views.paginated_elastic_view import PaginatedElasticSearchAPIView +from search.documents import EventDocument + +logger = structlog.get_logger(__name__) + + +def timeframe_query(timeframe: str) -> Optional[ESQuery]: + """Upcoming = not ended yet; past = already ended (dates compared to today, UTC). + + An event's end is its end date, or its start date when it has none. Events + with no dates at all match neither. + """ + if timeframe == "upcoming": + end, start = {"gte": "now/d"}, {"gte": "now/d"} + elif timeframe == "past": + end, start = {"lt": "now/d"}, {"lt": "now/d"} + else: + return None + return ESQ( + "bool", + should=[ + ESQ("range", end_date=end), + ESQ( + "bool", + must_not=[ESQ("exists", field="end_date")], + filter=[ESQ("range", start_date=start)], + ), + ], + minimum_should_match=1, + ) + + +class EventDocumentSerializer(serializers.ModelSerializer): + """Serializer for the Event search document.""" + + public_id = serializers.CharField() + url_slug = serializers.CharField() + # Elasticsearch hands dates back as datetimes; show them as plain dates. + start_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) + end_date = serializers.DateTimeField(format="%Y-%m-%d", allow_null=True) + is_individual_event = serializers.BooleanField() + # Luma cover image (https URL on images.lumacdn.com), null when unknown. + cover_url = serializers.CharField(default=None, allow_null=True) + + class OrganizationSerializer(serializers.Serializer): + name = serializers.CharField() + logo = serializers.CharField() + + class UserSerializer(serializers.Serializer): + name = serializers.CharField() + bio = serializers.CharField() + profile_picture = serializers.CharField() + + organization = OrganizationSerializer(allow_null=True) + user = UserSerializer(allow_null=True) + + class Meta: + model = Event + fields = [ + "id", + "name", + "public_id", + "url_slug", + "location", + "start_date", + "end_date", + "status", + "is_individual_event", + "cover_url", + "created", + "modified", + "organization", + "user", + ] + + +class SearchEvent(PaginatedElasticSearchAPIView): + """View for searching events.""" + + serializer_class = EventDocumentSerializer + document_class = EventDocument + permission_classes = [AllowAny] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.searchable_fields: List[str] + self.aggregations: Dict[str, str] + self.searchable_fields, self.aggregations = self.get_searchable_and_aggregations() + self.logger = structlog.get_logger(__name__) + + @trace_method( + name="get_searchable_and_aggregations", + attributes={"component": "search_event"}, + ) + def get_searchable_and_aggregations(self) -> Tuple[List[str], Dict[str, str]]: + """Searchable fields (name/location/owner) and the facet aggregations.""" + searchable_fields: List[str] = ["name", "location", "organization.name", "user.name"] + aggregations: Dict[str, str] = { + "status": "terms", + "location.raw": "terms", + "is_individual_event": "terms", + } + return searchable_fields, aggregations + + @trace_method(name="add_aggregations", attributes={"component": "search_event"}) + def add_aggregations(self, search: Search) -> Search: + """Add the facet aggregations to the search query.""" + for aggregation_field in self.aggregations: + search.aggs.bucket( + aggregation_field.replace(".raw", ""), + self.aggregations[aggregation_field], + field=aggregation_field, + ) + return search + + @trace_method(name="generate_q_expression", attributes={"component": "search_event"}) + def generate_q_expression(self, query: str) -> Optional[Union[ESQuery, List[ESQuery]]]: + """Analysed text query over the searchable fields (api/utils/search_query.py).""" + return text_query(query, self.searchable_fields, nested=nested_paths(self.document_class)) + + @trace_method(name="add_filters", attributes={"component": "search_event"}) + def add_filters(self, filters: Dict[str, str], search: Search) -> Search: + """Apply location / owner-type / timeframe filters.""" + for filter_key in filters: + if filter_key == "location": + filter_values = filters[filter_key].split(",") + search = search.filter("terms", **{"location.raw": filter_values}) + elif filter_key == "is_individual_event": + value = filters[filter_key].lower() == "true" + search = search.filter("term", is_individual_event=value) + elif filter_key == "timeframe": + timeframe = timeframe_query(filters[filter_key]) + if timeframe is not None: + search = search.filter(timeframe) + elif filter_key == "status": + search = search.filter("term", **{filter_key: filters[filter_key]}) + return search + + @trace_method(name="add_sort", attributes={"component": "search_event"}) + def add_sort(self, sort: str, search: Search, order: str) -> Search: + """Apply a sort mode (alphabetical / recent / created / start_date).""" + if sort == "alphabetical": + search = search.sort({"name.raw": {"order": order}}) + elif sort == "recent": + search = search.sort({"modified": {"order": order}}) + elif sort == "created": + search = search.sort({"created": {"order": order}}) + elif sort == "start_date": + search = search.sort({"start_date": {"order": order, "missing": "_last"}}) + return search diff --git a/api/views/search_publication.py b/api/views/search_publication.py index 714a436f..21ee5159 100644 --- a/api/views/search_publication.py +++ b/api/views/search_publication.py @@ -25,6 +25,9 @@ class PublicationDocumentSerializer(serializers.ModelSerializer): """Serializer for the Publication search document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + resource_type = serializers.CharField(allow_blank=True) sectors = serializers.ListField() geographies = serializers.ListField() @@ -45,6 +48,8 @@ class Meta: model = Publication fields = [ "id", + "public_id", + "url_slug", "title", "description", "status", diff --git a/api/views/search_unified.py b/api/views/search_unified.py index 1323f5ec..63707d0a 100644 --- a/api/views/search_unified.py +++ b/api/views/search_unified.py @@ -39,6 +39,8 @@ class UnifiedSearchResultSerializer(serializers.Serializer): title = serializers.CharField() description = serializers.CharField() slug = serializers.CharField(required=False) + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) created = serializers.DateTimeField(required=False) modified = serializers.DateTimeField(required=False) created_at = serializers.DateTimeField(required=False) @@ -85,6 +87,15 @@ class UserSerializer(serializers.Serializer): resource_type = serializers.CharField(required=False) is_individual_publication = serializers.BooleanField(required=False) + # Event specific + public_id = serializers.CharField(required=False) + url_slug = serializers.CharField(required=False) + location = serializers.CharField(required=False, allow_blank=True) + start_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) + end_date = serializers.DateTimeField(format="%Y-%m-%d", required=False, allow_null=True) + is_individual_event = serializers.BooleanField(required=False) + cover_url = serializers.CharField(required=False, allow_null=True) + # Collaborative specific is_individual_collaborative = serializers.BooleanField(required=False) website = serializers.CharField(required=False) @@ -149,6 +160,12 @@ def _get_index_names(self, types_list: List[str]) -> List[str]: ) index_names.append(collaborative_index) + if "event" in types_list: + event_index = settings.ELASTICSEARCH_INDEX_NAMES.get( + "search.documents.event_document", "event" + ) + index_names.append(event_index) + if "publisher" in types_list: org_publisher_index = settings.ELASTICSEARCH_INDEX_NAMES.get( "search.documents.publisher_document.OrganizationPublisherDocument", @@ -327,6 +344,8 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["type"] = "publication" elif "collaborative" in index_name: result["type"] = "collaborative" + elif "event" in index_name: + result["type"] = "event" elif "publisher" in index_name: result["type"] = "publisher" else: @@ -356,6 +375,13 @@ def _normalize_result(self, hit: Any) -> Dict[str, Any]: result["description"] = result.get("summary", "") if "title" not in result: result["title"] = "" + elif result["type"] == "event": + result["title"] = result.get("name", "") + result["description"] = result.get("location") or "" + result.setdefault("cover_url", None) + result.setdefault("tags", []) + result.setdefault("sectors", []) + result.setdefault("geographies", []) elif result["type"] == "publisher": if "name" in result: result["title"] = result.get("name", "") @@ -441,6 +467,8 @@ def perform_unified_search( aggregations["types"]["publication"] = bucket["doc_count"] elif "collaborative" in index_name: aggregations["types"]["collaborative"] = bucket["doc_count"] + elif "event" in index_name: + aggregations["types"]["event"] = bucket["doc_count"] elif "publisher" in index_name: # Combine both organization and user publisher counts if "publisher" not in aggregations["types"]: @@ -464,7 +492,7 @@ def _generate_unified_cache_key(self, request: Any) -> str: "page": request.GET.get("page", "1"), "size": request.GET.get("size", "10"), "types": request.GET.get( - "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,event,publisher" ), "filters": str(sorted(request.GET.dict().items())), "version": str(cache.get(SEARCH_CACHE_VERSION_KEY, 0)), @@ -484,7 +512,7 @@ def get(self, request: Any) -> Response: page: int = int(request.GET.get("page", 1)) size: int = int(request.GET.get("size", 10)) entity_types: str = request.GET.get( - "types", "dataset,usecase,aimodel,publication,collaborative,publisher" + "types", "dataset,usecase,aimodel,publication,collaborative,event,publisher" ) # Which entity types to search types_list = [t.strip() for t in entity_types.split(",")] diff --git a/api/views/search_usecase.py b/api/views/search_usecase.py index a0edeaf7..a6e9a639 100644 --- a/api/views/search_usecase.py +++ b/api/views/search_usecase.py @@ -61,6 +61,9 @@ def to_internal_value(self, data: Dict[str, Any]) -> Dict[str, Any]: class UseCaseDocumentSerializer(serializers.ModelSerializer): """Serializer for UseCase document.""" + public_id = serializers.CharField(required=False, allow_null=True) + url_slug = serializers.CharField(required=False, allow_null=True) + metadata = UseCaseMetadataSerializer(many=True) tags = serializers.ListField() logo = serializers.CharField() @@ -109,6 +112,8 @@ class Meta: model = UseCase fields = [ "id", + "public_id", + "url_slug", "title", "summary", "slug", diff --git a/authorization/migrations/0005_profile_picture_storage.py b/authorization/migrations/0005_profile_picture_storage.py new file mode 100644 index 00000000..61d9507e --- /dev/null +++ b/authorization/migrations/0005_profile_picture_storage.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0.4 on 2026-10-12 20:25 + +from django.db import migrations, models + +import api.utils.file_paths +import api.utils.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ("authorization", "0004_userconsent"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="profile_picture", + field=models.ImageField( + blank=True, + null=True, + storage=api.utils.storage.public_storage, + upload_to=api.utils.file_paths._user_profile_image_directory_path, + ), + ), + ] diff --git a/authorization/models.py b/authorization/models.py index 7a295002..85927475 100644 --- a/authorization/models.py +++ b/authorization/models.py @@ -3,6 +3,7 @@ from django.utils.text import slugify from api.utils.file_paths import _user_profile_image_directory_path +from api.utils.storage import public_storage from authorization.consent import UserConsent @@ -46,7 +47,10 @@ class User(AbstractUser): # Additional user profile fields bio = models.TextField(blank=True, null=True) profile_picture = models.ImageField( - upload_to=_user_profile_image_directory_path, blank=True, null=True + upload_to=_user_profile_image_directory_path, + storage=public_storage, + blank=True, + null=True, ) github_profile = models.URLField(blank=True, null=True) linkedin_profile = models.URLField(blank=True, null=True) diff --git a/authorization/permissions.py b/authorization/permissions.py index ce818e9b..ca31d877 100644 --- a/authorization/permissions.py +++ b/authorization/permissions.py @@ -5,8 +5,9 @@ from strawberry.permission import BasePermission from strawberry.types import Info -from api.models import Dataset, Organization, Publication -from api.utils.enums import PublicationStatus +from api.models import Dataset, Event, Organization, Publication +from api.utils.enums import EventStatus, PublicationStatus +from api.utils.public_links import parse_public_id from authorization.models import DatasetPermission, OrganizationMembership, Role # Roles that may publish/unpublish and edit an org-owned publication, by name. @@ -392,15 +393,6 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True -class ResourcePermissionGraphQL(HasOrganizationRoleGraphQL): # type: ignore[misc] - """ - Permission class specifically for Resource objects. - """ - - def _get_organization(self, obj: Any) -> Optional[Organization]: - return obj.dataset.organization if hasattr(obj, "dataset") else None # type: ignore[attr-defined,no-any-return] - - class PublishDatasetPermission(BasePermission): """Permission class for publishing a dataset. Checks if the user has permission to publish the dataset. @@ -581,6 +573,14 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: return True +def find_publication(publication_id: Any, public_id: Optional[str]) -> Optional[Publication]: + """The publication for a UUID, or for a public link (``urlSlug`` or ``publicId``).""" + if publication_id: + return Publication.objects.filter(id=publication_id).first() + parsed = parse_public_id(public_id) + return Publication.objects.filter(public_id=parsed).first() if parsed else None + + class AllowPublishedPublications(BasePermission): # type: ignore[misc] """Read gate for a single publication — mirrors AllowPublishedDatasets. @@ -593,11 +593,11 @@ class AllowPublishedPublications(BasePermission): # type: ignore[misc] def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: request = info.context publication_id = kwargs.get("publication_id") + public_id = kwargs.get("public_id") - if publication_id: - try: - publication = Publication.objects.get(id=publication_id) - except Publication.DoesNotExist: + if publication_id or public_id: + publication = find_publication(publication_id, public_id) + if publication is None: return True # Let the resolver return a clean not-found. if publication.status == PublicationStatus.PUBLISHED.value: @@ -622,3 +622,137 @@ def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: if source.status == PublicationStatus.PUBLISHED.value: return True return bool(getattr(request, "user", None) and request.user.is_authenticated) + + +# --------------------------------------------------------------------------- +# Event permissions — same ownership rules as publications. +# --------------------------------------------------------------------------- + + +def _resolve_event_id(kwargs: Any) -> Optional[Any]: + """Pull the target event id out of a mutation's arguments.""" + event_id = kwargs.get("event_id") + if event_id: + return event_id + payload = kwargs.get("input") + if payload is not None and getattr(payload, "id", None): + return payload.id + return None + + +def _user_manages_event(user: Any, event: Event, operation: str) -> bool: + """Whether a user may perform ``operation`` on an event.""" + if user.is_superuser: + return True + if event.user and event.user == user: + return True + if not event.organization: + return False + + membership = OrganizationMembership.objects.filter( + user=user, organization=event.organization + ).first() + if not membership: + return False + + role = membership.role + if operation == "publish": + return role.name in PUBLICATION_MANAGER_ROLE_NAMES + if operation == "delete": + return role.can_delete + return role.can_change + + +class EventPermissionGraphQL(BasePermission): # type: ignore[misc] + """Base event mutation permission — keys on the event id in kwargs.""" + + message = "You don't have permission to modify this event" + operation = "change" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + event_id = _resolve_event_id(kwargs) + if not event_id: + return False + + try: + event = Event.objects.get(id=event_id) + except Event.DoesNotExist: + return False + + return _user_manages_event(user, event, self.operation) + + +class ChangeEventPermission(EventPermissionGraphQL): + operation = "change" + + +class DeleteEventPermission(EventPermissionGraphQL): + message = "You don't have permission to delete this event" + operation = "delete" + + +class PublishEventPermission(EventPermissionGraphQL): + message = "You don't have permission to publish this event" + operation = "publish" + + +class CreateEventPermission(BasePermission): # type: ignore[misc] + """Any authenticated user may create an individual event. + + Creating inside an organization requires the ``add`` role. + """ + + message = "You don't have permission to create an event" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + user = info.context.user + if not getattr(user, "is_authenticated", False): + return False + + organization = info.context.context.get("organization") + if organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=organization + ).first() + return bool(membership and membership.role.can_add) + + return True + + +class AllowPublishedEvents(BasePermission): # type: ignore[misc] + """A published event is world-readable. A draft is visible to its owner.""" + + message = "You need to be authenticated to access non-published events" + + def has_permission(self, source: Any, info: Info, **kwargs: Any) -> bool: + request = info.context + event_id = kwargs.get("event_id") + + if event_id: + try: + event = Event.objects.get(id=event_id) + except Event.DoesNotExist: + return True + + if event.status == EventStatus.PUBLISHED.value: + return True + + user = request.user + if not user.is_authenticated: + return False + if user.is_superuser: + return True + if event.user and event.user == user: + return True + if event.organization: + membership = OrganizationMembership.objects.filter( + user=user, organization=event.organization + ).first() + return bool(membership and membership.role.can_view) + return False + + return bool(getattr(request, "user", None) and request.user.is_authenticated) diff --git a/dataspace_sdk/resources/datasets.py b/dataspace_sdk/resources/datasets.py index c3ae3a62..3c4d8226 100644 --- a/dataspace_sdk/resources/datasets.py +++ b/dataspace_sdk/resources/datasets.py @@ -1,6 +1,6 @@ """Dataset resource client for DataSpace SDK.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterator, List, Optional from dataspace_sdk.base import BaseAPIClient @@ -169,6 +169,7 @@ def list_all( license created updated + datasetType organization { id name @@ -177,6 +178,31 @@ def list_all( id value } + sectors { + id + name + } + promptMetadata + resources { + id + name + noOfEntries + fileDetails { + format + size + } + schema { + format + description + fieldName + } + promptDetails { + promptFormat + hasSystemPrompt + hasExampleResponses + promptCount + } + } } } """ @@ -617,3 +643,176 @@ def update_prompt_metadata( result: Dict[str, Any] = response.get("data", {}).get("updatePromptMetadata", {}) return result + + # ------------------------------------------------------------------ + # Indexed data access (data_db) + # ------------------------------------------------------------------ + + @staticmethod + def _build_data_params( + filters: Optional[Dict[str, Any]], + columns: Optional[List[str]], + order_by: Optional[List[str]], + limit: int, + offset: int, + count: Optional[bool], + ) -> Dict[str, Any]: + """Translate Pythonic kwargs into the ``GET /data/`` query-string form. + + ``filters`` is a flat dict using the same ``col`` / ``col__op`` keys as + the server. List values are passed through (requests will emit one + ``key=v`` pair per entry, used by ``__in`` / ``__nin``). + """ + params: Dict[str, Any] = {"limit": int(limit), "offset": int(offset)} + if columns: + params["columns"] = ",".join(columns) + if order_by: + params["order_by"] = ",".join(order_by) + if count is not None: + params["count"] = "true" if count else "false" + if filters: + for k, v in filters.items(): + if isinstance(v, (list, tuple)): + params[k] = list(v) + elif isinstance(v, bool): + params[k] = "true" if v else "false" + else: + params[k] = v + return params + + def get_resource_data( + self, + resource_id: str, + filters: Optional[Dict[str, Any]] = None, + columns: Optional[List[str]] = None, + order_by: Optional[List[str]] = None, + limit: int = 100, + offset: int = 0, + count: bool = True, + ) -> Dict[str, Any]: + """Fetch indexed (saved in ``data_db``) data for a single resource. + + Args: + resource_id: UUID of the resource (must have an indexed table). + filters: Column-level filters. Keys are either ``"col"`` (equality) + or ``"col__op"`` where op is one of: ``eq, ne, gt, gte, lt, + lte, in, nin, contains, icontains, startswith, istartswith, + endswith, iendswith, isnull, notnull``. + columns: Subset of columns to project. ``None`` returns all. + order_by: Columns to sort by. Prefix with ``-`` for DESC. + limit: Max rows to return (server caps at 10000). + offset: Number of rows to skip. + count: If ``True``, the response includes total matching row count. + + Returns: + A dict with ``columns``, ``rows``, ``total``, ``limit``, + ``offset``, ``available_columns``, ``resource_id``, + ``dataset_id``, and ``max_limit``. + """ + params = self._build_data_params(filters, columns, order_by, limit, offset, count) + return self.get(f"/api/resources/{resource_id}/data/", params=params) + + def get_dataset_data( + self, + dataset_id: str, + resource_id: Optional[str] = None, + filters: Optional[Dict[str, Any]] = None, + columns: Optional[List[str]] = None, + order_by: Optional[List[str]] = None, + limit: int = 100, + offset: int = 0, + count: bool = True, + ) -> Dict[str, Any]: + """Fetch indexed data for a dataset. + + By default operates on the dataset's first indexed (tabular) resource. + Pass ``resource_id`` to target a specific resource within the dataset. + Filtering / column / ordering semantics are identical to + :meth:`get_resource_data`. + """ + params = self._build_data_params(filters, columns, order_by, limit, offset, count) + if resource_id: + params["resource_id"] = resource_id + return self.get(f"/api/datasets/{dataset_id}/data/", params=params) + + def get_prompt_data( + self, + dataset_id: str, + resource_id: Optional[str] = None, + filters: Optional[Dict[str, Any]] = None, + columns: Optional[List[str]] = None, + order_by: Optional[List[str]] = None, + limit: int = 100, + offset: int = 0, + count: bool = True, + prompt_contains: Optional[str] = None, + response_contains: Optional[str] = None, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + ) -> Dict[str, Any]: + """Fetch indexed data for a PROMPT-typed dataset. + + Same generic semantics as :meth:`get_dataset_data`, plus prompt-aware + shorthands that automatically map to the underlying prompt/response/ + length columns when present: + + Args: + prompt_contains: Substring (case-insensitive) match on the prompt + column (auto-detects ``prompt``/``input``/``instruction``/ + ``question``). + response_contains: Substring match on the response column + (auto-detects ``response``/``completion``/``answer``/ + ``output``). + min_length / max_length: Bounds on the length column + (auto-detects ``length``/``prompt_length``/``tokens``/ + ``token_count``). + + The response includes ``prompt_column``, ``response_column``, and + ``length_column`` indicating what was auto-detected. + """ + params = self._build_data_params(filters, columns, order_by, limit, offset, count) + if resource_id: + params["resource_id"] = resource_id + if prompt_contains is not None: + params["prompt_contains"] = prompt_contains + if response_contains is not None: + params["response_contains"] = response_contains + if min_length is not None: + params["min_length"] = int(min_length) + if max_length is not None: + params["max_length"] = int(max_length) + return self.get(f"/api/datasets/{dataset_id}/prompts/", params=params) + + def iter_resource_data( + self, + resource_id: str, + filters: Optional[Dict[str, Any]] = None, + columns: Optional[List[str]] = None, + order_by: Optional[List[str]] = None, + batch_size: int = 1000, + ) -> Iterator[Dict[str, Any]]: + """Yield rows as dicts, paging through the entire filtered result set. + + Each yielded item is a ``{column: value}`` mapping. ``batch_size`` is + capped at 10000 by the server. + """ + offset = 0 + while True: + page = self.get_resource_data( + resource_id=resource_id, + filters=filters, + columns=columns, + order_by=order_by, + limit=batch_size, + offset=offset, + count=False, + ) + cols: List[str] = page.get("columns", []) or [] + rows: List[List[Any]] = page.get("rows", []) or [] + if not rows: + return + for row in rows: + yield dict(zip(cols, row)) + if len(rows) < batch_size: + return + offset += len(rows) diff --git a/docker-compose.hotreload.yml b/docker-compose.hotreload.yml index d8fce17a..a25508f5 100644 --- a/docker-compose.hotreload.yml +++ b/docker-compose.hotreload.yml @@ -7,3 +7,6 @@ services: command: uvicorn DataSpace.asgi:application --host 0.0.0.0 --port 8000 --reload volumes: - .:/code + + networks: !override + - default \ No newline at end of file diff --git a/docs/dataset_data_api.md b/docs/dataset_data_api.md new file mode 100644 index 00000000..8857b105 --- /dev/null +++ b/docs/dataset_data_api.md @@ -0,0 +1,267 @@ +# Indexed Dataset Data API + +This document describes the HTTP endpoints and SDK methods for fetching the +*indexed tabular data* that DataSpace stores in the `data_db` PostgreSQL +database. When a CSV / XLSX / Parquet / JSON resource is uploaded, its rows are +indexed into a per-resource table so they can be queried, filtered, and +streamed without re-downloading the source file. How those tables are built, +named, replaced and cleaned up: `api/utils/data_indexing_architecture.md`. + +## Overview + +| Layer | Surface | +|-------|---------| +| Backend utility | `api.utils.data_indexing.fetch_resource_data(...)` | +| HTTP API | `GET /api/resources//data/`, `GET /api/datasets//data/`, `GET /api/datasets//prompts/` | +| Python SDK | `DatasetClient.get_resource_data(...)`, `get_dataset_data(...)`, `get_prompt_data(...)`, `iter_resource_data(...)` | + +All three endpoints share the same query-parameter contract. The prompt +endpoint adds prompt-specific shorthands. + +## Permissions + +- **PUBLISHED** datasets are publicly readable. +- **DRAFT / ARCHIVED** datasets require the requesting user to be the dataset + owner, a superuser, or a member of the dataset's organization. + +## HTTP API + +### `GET /api/resources//data/` + +Returns indexed data for a single resource. + +### `GET /api/datasets//data/` + +Returns indexed data for a dataset. Defaults to the dataset's first indexed +resource. Use `?resource_id=` to target a specific resource. + +### `GET /api/datasets//prompts/` + +Same semantics as `/data/`, but the dataset must be of `dataset_type=PROMPT` +and the response includes the auto-detected prompt / response / length column +names. Convenience filters: + +| Param | Maps to | +|-------|---------| +| `prompt_contains=` | `__icontains=` | +| `response_contains=` | `__icontains=` | +| `min_length=` | `__gte=` | +| `max_length=` | `__lte=` | + +Auto-detected columns (case-insensitive, first match wins): + +- prompt: `prompt`, `input`, `instruction`, `question` +- response: `response`, `completion`, `answer`, `output` +- length: `length`, `prompt_length`, `tokens`, `token_count` + +If a candidate column is not present in the resource schema, the corresponding +shorthand is silently ignored. You can always fall back to the explicit +`__` form. + +### Query parameters + +Reserved (not interpreted as filters): + +| Param | Default | Notes | +|-------|---------|-------| +| `columns` | all | Comma-separated list of columns to project. | +| `limit` | `100` | Capped at `10000`. | +| `offset` | `0` | | +| `order_by` | none | Comma-separated. Prefix with `-` for DESC. | +| `count` | `true` | Set `false` to skip the `SELECT COUNT(*)` round-trip. | +| `resource_id` | first indexed | Only on `/datasets//data/` and `/prompts/`. | + +Any other query param is treated as a column filter. + +### Filter operators + +Filters use Django-ORM-style suffixes: `?__=`. Without a +suffix, equality is assumed: `?=`. + +| Operator | SQL | Notes | +|----------|-----|-------| +| `eq` (default) | `=` | | +| `ne` | `<>` | | +| `gt`, `gte`, `lt`, `lte` | `>`, `>=`, `<`, `<=` | | +| `in` | `= ANY(...)` | Repeat the param: `?col__in=a&col__in=b` (or `?col__in=a,b`). | +| `nin` | `<> ALL(...)` | Same shape as `in`. | +| `contains` / `icontains` | `LIKE` / `ILIKE` `'%v%'` | | +| `startswith` / `istartswith` | `LIKE` / `ILIKE` `'v%'` | | +| `endswith` / `iendswith` | `LIKE` / `ILIKE` `'%v'` | | +| `isnull` | `IS NULL` (truthy) / `IS NOT NULL` (falsy) | Value is parsed as bool. | +| `notnull` | inverse of `isnull` | | + +Unknown columns or unknown operators return HTTP **400** with a +`{"error": "..."}` body. All identifiers are quoted via `psycopg2.sql`; values +are bound as parameters — there is no string concatenation into the SQL. + +### Response shape + +```json +{ + "resource_id": "f1e2...", + "dataset_id": "abcd...", + "available_columns": ["id", "name", "price", "category"], + "max_limit": 10000, + "columns": ["id", "name"], + "rows": [[1, "alpha"], [2, "beta"]], + "total": 87, + "limit": 100, + "offset": 0 +} +``` + +The prompt endpoint additionally returns: + +```json +{ + "dataset_type": "PROMPT", + "prompt_column": "prompt", + "response_column": "response", + "length_column": "tokens" +} +``` + +Set `?count=false` to avoid the count query for large tables; `total` will be +`null`. + +### Examples + +```bash +# Books over $10, sorted by descending price, page 2 +curl "https://api.example.com/api/resources//data/?\ +category=books&price__gte=10&order_by=-price&limit=50&offset=50" + +# Multiple categories +curl "https://api.example.com/api/resources//data/?\ +category__in=books&category__in=media" + +# Prompt dataset: long English translation prompts +curl "https://api.example.com/api/datasets//prompts/?\ +prompt_contains=translate&min_length=50&language=en" +``` + +## Python SDK + +```python +from dataspace_sdk import DataSpaceClient + +client = DataSpaceClient( + base_url="https://dataspace.civicdatalab.in", + keycloak_url="https://opub-kc.civicdatalab.in", + keycloak_realm="DataSpace", + keycloak_client_id="dataspace", +) +client.login(username="...", password="...") +``` + +### `get_resource_data` + +```python +page = client.datasets.get_resource_data( + resource_id="f1e2...", + filters={ + "price__gte": 10, + "category__in": ["books", "media"], + "is_active": True, + }, + columns=["id", "title", "price"], + order_by=["-price", "title"], + limit=200, + offset=0, + count=True, +) +print(page["total"], len(page["rows"])) +``` + +### `get_dataset_data` + +Same parameters as `get_resource_data`, plus an optional `resource_id`. +Without `resource_id`, the dataset's first indexed resource is used. + +```python +page = client.datasets.get_dataset_data( + dataset_id="abcd...", + resource_id="optional-uuid", + filters={"region": "south"}, +) +``` + +### `get_prompt_data` + +Adds prompt-aware shorthands on top of the generic interface: + +```python +page = client.datasets.get_prompt_data( + dataset_id="abcd...", + prompt_contains="translate", + response_contains="bonjour", + min_length=20, + max_length=400, + filters={"language": "fr"}, + columns=["prompt", "response", "tokens"], + order_by=["-tokens"], +) +print(page["prompt_column"], page["response_column"], page["length_column"]) +``` + +### `iter_resource_data` — streaming all rows + +Transparently pages through the entire filtered result set, yielding each row +as a `{column: value}` dict. The server caps `batch_size` at `10000`. + +```python +for row in client.datasets.iter_resource_data( + resource_id="f1e2...", + filters={"is_active": True}, + columns=["id", "title", "price"], + batch_size=2000, +): + process(row) +``` + +## Backend utility + +When you need to fetch indexed data from inside the Django process (e.g. a +GraphQL resolver or background task), call the underlying utility directly: + +```python +from api.models import Resource +from api.utils.data_indexing import fetch_resource_data, DataFetchError + +resource = Resource.objects.get(id=resource_id) +try: + result = fetch_resource_data( + resource=resource, + filters={"price__gte": 10}, + columns=["id", "title", "price"], + order_by=["-price"], + limit=100, + offset=0, + count=True, + ) +except DataFetchError as e: + # Unknown column / no indexed table / etc. + raise +``` + +The utility validates every column against `ResourceSchema` (or the live +`information_schema` if no schema rows exist) and uses parameterised queries +exclusively — passing a malicious column name returns `DataFetchError`, +never a SQL injection. + +## Safety notes + +- Identifiers are quoted via `psycopg2.sql.Identifier`; values are passed as + query parameters. There is no string interpolation of user input into SQL. +- `statement_timeout` is set to **10 seconds** on every fetch. +- `limit` is clamped to **10000** rows. Use `iter_resource_data` to stream + larger result sets. +- The `data_db` connection is read-only from this layer's perspective — the + utility never executes anything other than `SELECT` / `SET statement_timeout`. + +## Related + +- [SDK overview](sdk/OVERVIEW.md) +- [SDK quick start](sdk/QUICKSTART.md) +- [Unified search API](unified_search_api.md) diff --git a/docs/sdk/README.md b/docs/sdk/README.md index aabd9c2e..bfbe1fbc 100644 --- a/docs/sdk/README.md +++ b/docs/sdk/README.md @@ -206,6 +206,50 @@ org_datasets = client.datasets.get_organization_datasets( ) ``` +### Fetch Indexed Dataset Data (filterable) + +For datasets whose resources have been indexed into `data_db` (CSV/XLSX/etc.), +you can query the underlying rows with column-level filters, projection, and +ordering. See the dedicated guide: [dataset_data_api.md](../dataset_data_api.md). + +```python +# Per-resource fetch +page = client.datasets.get_resource_data( + resource_id="f1e2...", + filters={"price__gte": 10, "category__in": ["books", "media"]}, + columns=["id", "title", "price"], + order_by=["-price"], + limit=200, +) +print(page["total"], len(page["rows"])) + +# Per-dataset fetch (defaults to first indexed resource) +page = client.datasets.get_dataset_data( + dataset_id="abcd...", + filters={"region": "south"}, +) + +# Prompt datasets — extra prompt-aware shorthands +page = client.datasets.get_prompt_data( + dataset_id="abcd...", + prompt_contains="translate", + min_length=20, + filters={"language": "fr"}, +) + +# Stream all matching rows as dicts +for row in client.datasets.iter_resource_data( + resource_id="f1e2...", + filters={"is_active": True}, + batch_size=2000, +): + process(row) +``` + +Supported filter operators (Django-style suffixes): `eq, ne, gt, gte, lt, lte, +in, nin, contains, icontains, startswith, istartswith, endswith, iendswith, +isnull, notnull`. + ## Working with AI Models ### Search AI Models diff --git a/docs/version_detection_system.md b/docs/version_detection_system.md deleted file mode 100644 index 66fbd99f..00000000 --- a/docs/version_detection_system.md +++ /dev/null @@ -1,140 +0,0 @@ -# Intelligent Version Detection System - -## Overview - -The DataSpace platform implements an version detection system that automatically determines the appropriate version increment (major, minor, or patch) when resources are updated. This document explains the technical implementation, triggering mechanisms, and version classification logic. - -## Version Increment Types - -The system follows semantic versioning principles (X.Y.Z): - -1. **Major Version (X)**: Breaking changes or significant structural modifications -2. **Minor Version (Y)**: Publishing Dataset or significant data changes that maintain compatibility -3. **Patch Version (Z)**: Small fixes, corrections, or minimal data changes - -## Triggering Mechanisms - -Version changes are automatically triggered by the following events: - -1. **Resource File Updates**: When a resource file is updated through the API -2. **Dataset Publication**: When a dataset containing resources is published -3. **Manual Triggers**: Through management commands (e.g., `create_major_version`) - -## Technical Implementation - -### Components - -1. **Signal Handlers**: Detect changes to resources and trigger version detection -2. **Version Detection Utility**: Analyzes file changes to determine version increment type -3. **DVC Manager**: Handles version tracking, tagging, and remote storage - -### Signal Flow - -``` -ResourceFileDetails update → post_save signal → version_resource_with_dvc - → detect_version_change_type → _increment_version → Create ResourceVersion -``` - -## Change Detection Logic - -The system uses different strategies based on file type: - -### CSV/Tabular Files - -**Major Version** triggers: -- Schema changes (columns added/removed) -- Data type changes in columns - -**Minor Version** triggers: -- Row count changes > 10% -- Data changes in > 30% of cells - -**Patch Version** triggers: -- Small data changes (< 30% of cells) -- Minimal corrections - -### JSON Files - -**Major Version** triggers: -- Structure changes (keys added/removed) -- Data type changes - -**Minor Version** triggers: -- Significant value changes (> 30% of values) -- Array item additions/removals - -**Patch Version** triggers: -- Small value changes -- Formatting changes - -### XML Files - -**Major Version** triggers: -- Tag structure changes - -**Minor Version** triggers: -- Attribute changes - -**Patch Version** triggers: -- Content changes without structural modifications - -### Generic Files - -For non-structured files, the system uses file size differences: - -**Major Version** triggers: -- Size changes > 50% - -**Minor Version** triggers: -- Size changes between 10% and 50% - -**Patch Version** triggers: -- Size changes < 10% - -## Technical Details - -### Dependencies - -- **pandas**: For efficient tabular data comparison -- **DeepDiff**: For accurate JSON structure comparison -- **DVC**: For version tracking and storage - -### Performance Considerations - -- Large files (>100MB) use chunked processing -- Tabular comparisons use sampling for very large datasets -- Early return logic to avoid unnecessary processing - -## Example - -When a CSV resource is updated: - -1. The system detects the change through a Django signal -2. `detect_version_change_type` loads both versions of the file -3. The function compares schemas, data types, and values -4. Based on the extent of changes, it returns "major", "minor", or "patch" -5. The version is incremented accordingly (e.g., 1.2.3 → 1.3.0 for minor) -6. DVC tracks the new version with appropriate tags -7. A ResourceVersion record is created in the database - -## Management Commands - -The system includes management commands for manual version control: - -- `create_major_version`: Force a major version increment for a resource -- `setup_dvc`: Configure DVC repository and remotes - -## Error Handling - -The version detection system includes robust error handling to ensure that: - -1. Failed comparisons default to "minor" version changes -2. Temporary files are properly cleaned up -3. Errors are logged with detailed context -4. The system continues functioning even if analysis fails - -## Future Improvements - -- Support for more file formats -- Configurable thresholds for different file types -- Performance optimizations for very large datasets diff --git a/pytest.ini b/pytest.ini index b493e2f9..634db965 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,6 @@ python_functions = test_* testpaths = tests pythonpath = . django_find_project = true -addopts = -v --nomigrations --tb=short \ No newline at end of file +addopts = -v --nomigrations --tb=short +markers = + data_db: builds tables in the Postgres data database; needs TEST_DATA_DB_URL (see tests/data_index/conftest.py) diff --git a/requirements.txt b/requirements.txt index 31439945..83688671 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,9 @@ django-elasticsearch-dsl==8.0 django-elasticsearch-dsl-drf==0.22.5 django-nine==0.2.7 djangorestframework==3.14.0 -dvc==3.59.1 +# S3 media storage +django-storages[s3]==1.14.4 +boto3==1.43.108 elastic-transport==8.13.0 elasticsearch==8.13.0 elasticsearch-dsl==8.12.0 diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index 3f3aab65..aa9bc354 100755 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -134,7 +134,7 @@ $RELEASE migrate --noinput # exists" on its own, gets the correctness of both without either failure # mode. Keep this list in sync with DataSpace/settings.py's # ELASTICSEARCH_INDEX_NAMES if a new search document is ever added. -for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Organization authorization.User; do +for model in api.Dataset api.UseCase api.AIModel api.Publication api.Collaborative api.Event api.Organization authorization.User; do $RELEASE search_index --create --models "$model" -f || true done $RELEASE search_index --populate -f diff --git a/search/documents/__init__.py b/search/documents/__init__.py index 4c6a678a..b679fb3e 100644 --- a/search/documents/__init__.py +++ b/search/documents/__init__.py @@ -1,6 +1,7 @@ from search.documents.aimodel_document import AIModelDocument from search.documents.collaborative_document import CollaborativeDocument from search.documents.dataset_document import DatasetDocument +from search.documents.event_document import EventDocument from search.documents.publication_document import PublicationDocument from search.documents.publisher_document import ( OrganizationPublisherDocument, diff --git a/search/documents/aimodel_document.py b/search/documents/aimodel_document.py index 6c89bc2b..81c73c08 100644 --- a/search/documents/aimodel_document.py +++ b/search/documents/aimodel_document.py @@ -24,6 +24,8 @@ class AIModelDocument(Document): """Elasticsearch document for AIModel.""" # Basic fields with analyzers + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") name = fields.TextField( analyzer=ngram_analyser, fields=title_subfields(), diff --git a/search/documents/collaborative_document.py b/search/documents/collaborative_document.py index 0fcedeed..52f09682 100644 --- a/search/documents/collaborative_document.py +++ b/search/documents/collaborative_document.py @@ -32,6 +32,8 @@ class CollaborativeDocument(Document): """Elasticsearch document for Collaborative model.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") metadata = fields.NestedField( properties={ "value": KeywordField(multi=True), @@ -242,12 +244,12 @@ def prepare_organizations(self, instance: Collaborative) -> List[Dict[str, str]] def prepare_logo(self, instance: Collaborative) -> str: if instance.logo: - return str(instance.logo.path.replace("/code/files/", "")) + return str(instance.logo.url) return "" def prepare_cover_image(self, instance: Collaborative) -> str: if instance.cover_image: - return str(instance.cover_image.path.replace("/code/files/", "")) + return str(instance.cover_image.url) return "" def prepare_dataset_count(self, instance: Collaborative) -> int: diff --git a/search/documents/dataset_document.py b/search/documents/dataset_document.py index 03fd2a38..bce09ecd 100644 --- a/search/documents/dataset_document.py +++ b/search/documents/dataset_document.py @@ -53,6 +53,8 @@ class DatasetDocument(Document): status = fields.KeywordField() slug = fields.KeywordField() # Add slug field + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") tags = fields.TextField( attr="tags_indexing", diff --git a/search/documents/event_document.py b/search/documents/event_document.py new file mode 100644 index 00000000..565105dd --- /dev/null +++ b/search/documents/event_document.py @@ -0,0 +1,121 @@ +"""Elasticsearch document for Event. + +Indexes published events only: name, location, dates, link (``public_id`` / +``url_slug``), owner and ``cover_url``. The cover is looked up on Luma every +time the document is built (``prepare_cover_url``), including the deploy's +``search_index --populate``; it is display-only (not searchable). The Luma +embed itself is not indexed; the event page fetches it with ``eventBySlug``. +""" + +from typing import Any, Dict, List, Optional, Union + +from django_elasticsearch_dsl import Document, Index, KeywordField, fields + +from api.models.Event import Event +from api.models.Organization import Organization +from api.utils.enums import EventStatus +from api.utils.luma_cover import fetch_luma_cover +from authorization.models import User +from DataSpace import settings +from search.documents.analysers import ngram_analyser, title_subfields + +INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) +INDEX.settings(number_of_shards=1, number_of_replicas=0) + + +@INDEX.doc_type +class EventDocument(Document): + """Elasticsearch document for a published event.""" + + name = fields.TextField( + analyzer=ngram_analyser, + fields=title_subfields(), + ) + location = fields.TextField( + analyzer=ngram_analyser, + fields={"raw": KeywordField(multi=False)}, + ) + status = fields.KeywordField() + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") + is_individual_event = fields.BooleanField(attr="is_individual_event") + cover_url = fields.KeywordField(index=False) + + organization = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "logo": fields.TextField(analyzer=ngram_analyser), + } + ) + user = fields.NestedField( + properties={ + "name": fields.TextField(analyzer=ngram_analyser), + "bio": fields.TextField(analyzer=ngram_analyser), + "profile_picture": fields.TextField(analyzer=ngram_analyser), + } + ) + + def prepare_location(self, instance: Event) -> Optional[str]: + """Blank locations are left out, so they never show up as a filter option.""" + return instance.location or None + + def prepare_cover_url(self, instance: Event) -> Optional[str]: + """Cover image from Luma for the stored embed; None when Luma has none or fails.""" + embed = instance.luma_embed if isinstance(instance.luma_embed, dict) else {} + return fetch_luma_cover(embed.get("src")) + + def prepare_organization(self, instance: Event) -> Optional[Dict[str, str]]: + """Prepare the owning organization for indexing.""" + if instance.organization: + org = instance.organization + return {"name": org.name, "logo": org.logo.url if org.logo else ""} + return None + + def prepare_user(self, instance: Event) -> Optional[Dict[str, str]]: + """Prepare the owning user for indexing.""" + if instance.user: + user = instance.user + return { + "name": user.full_name, + "bio": user.bio or "", + "profile_picture": (user.profile_picture.url if user.profile_picture else ""), + } + return None + + def should_index_object(self, obj: Any) -> bool: + """Only PUBLISHED events are indexed — drafts never reach search.""" + return bool(obj.status == EventStatus.PUBLISHED.value) + + def save(self, *args: Any, **kwargs: Any) -> None: + """Index a published event, or drop it from the index otherwise.""" + if self.should_index_object(self.to_dict()): # type: ignore + super().save(*args, **kwargs) + else: + self.delete(ignore=404) + + def get_queryset(self) -> Any: + """Only published events are populated into the index.""" + return super(EventDocument, self).get_queryset().filter(status=EventStatus.PUBLISHED) + + def get_instances_from_related( + self, related_instance: Union[Organization, User] + ) -> Optional[List[Event]]: + """Re-index an owner's events when the owner's name or picture changes.""" + if isinstance(related_instance, (Organization, User)): + return list(related_instance.events.all()) + return None + + class Django: + """Django model configuration.""" + + model = Event + + fields = [ + "id", + "start_date", + "end_date", + "created", + "modified", + ] + + related_models = [Organization, User] diff --git a/search/documents/publication_document.py b/search/documents/publication_document.py index 5b5e04a8..f3c55558 100644 --- a/search/documents/publication_document.py +++ b/search/documents/publication_document.py @@ -29,6 +29,8 @@ class PublicationDocument(Document): """Elasticsearch document for a published Resource.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") title = fields.TextField( analyzer=ngram_analyser, fields=title_subfields(), diff --git a/search/documents/usecase_document.py b/search/documents/usecase_document.py index 4b2b1ea4..f7872c90 100644 --- a/search/documents/usecase_document.py +++ b/search/documents/usecase_document.py @@ -24,6 +24,8 @@ class UseCaseDocument(Document): """Elasticsearch document for UseCase model.""" + public_id = fields.KeywordField() + url_slug = fields.KeywordField(attr="url_slug") metadata = fields.NestedField( properties={ "value": KeywordField(multi=True), @@ -222,7 +224,7 @@ def prepare_organizations(self, instance: UseCase) -> List[Dict[str, str]]: def prepare_logo(self, instance: UseCase) -> str: """Prepare logo URL for indexing.""" if instance.logo: - return str(instance.logo.path.replace("/code/files/", "")) + return str(instance.logo.url) return "" def should_index_object(self, obj: UseCase) -> bool: diff --git a/search/urls.py b/search/urls.py index 7e663078..0b8ad4c4 100644 --- a/search/urls.py +++ b/search/urls.py @@ -32,4 +32,4 @@ # urlpatterns = [ # path('', include(router.urls)), -# ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +# ] diff --git a/tests/data_index/__init__.py b/tests/data_index/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data_index/conftest.py b/tests/data_index/conftest.py new file mode 100644 index 00000000..7703b854 --- /dev/null +++ b/tests/data_index/conftest.py @@ -0,0 +1,111 @@ +"""Shared setup for the data-index tests. + +Tests marked ``data_db`` build real tables in a Postgres data database. They +need TEST_DATA_DB_URL (see tests/test_settings.py), for example: + + docker run -d --name dataspace-test-data-db -e POSTGRES_PASSWORD=postgres \ + -p 55432:5432 postgres:16 + export TEST_DATA_DB_URL=postgres://postgres:postgres@localhost:55432/postgres + +Without it they are skipped, unless REQUIRE_DATA_DB_TESTS=1 (set in CI), in +which case the run fails: a green run that skipped them proves nothing. + +The main database stays SQLite, so row locks are no-ops here; the lock order is +proven by the flow tests and its effect by the journey script. +""" + +import os +import types +from typing import Any, Callable, List, Optional + +import pytest +from django.conf import settings +from django.core.files.base import ContentFile +from django.db import connections + +from api.models import Dataset, Organization, Resource, ResourceFileDetails +from authorization.models import User +from tests.data_index.support import ( + DATA_DB, + fixture_bytes, + list_public_tables, + table_comment, + table_exists, + table_rows, +) + + +def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item]) -> None: + if DATA_DB in settings.DATABASES: + return + data_db_items = [item for item in items if item.get_closest_marker("data_db")] + if not data_db_items: + return + if os.environ.get("REQUIRE_DATA_DB_TESTS") == "1": + raise pytest.UsageError( + f"{len(data_db_items)} data_db tests would be skipped: set TEST_DATA_DB_URL " + "(REQUIRE_DATA_DB_TESTS=1 forbids skipping them)." + ) + skip = pytest.mark.skip(reason="needs TEST_DATA_DB_URL (a Postgres data database)") + for item in data_db_items: + item.add_marker(skip) + + +@pytest.fixture(autouse=True) +def empty_data_db_after_test(request: pytest.FixtureRequest, django_db_blocker: Any) -> Any: + """Drop every table a data_db test left behind. + + pytest-django only empties tables of models routed to a database, and none + are routed to the data database, so without this each test would see the + tables of the tests before it. + """ + yield + if not request.node.get_closest_marker("data_db") or DATA_DB not in settings.DATABASES: + return + with django_db_blocker.unblock(): + for name in list_public_tables(): + with connections[DATA_DB].cursor() as cursor: + cursor.execute(f'DROP TABLE IF EXISTS "{name}" CASCADE') + + +@pytest.fixture +def tables() -> types.SimpleNamespace: + """Read-only peeks into the test data database.""" + return types.SimpleNamespace( + names=list_public_tables, exists=table_exists, rows=table_rows, comment=table_comment + ) + + +@pytest.fixture +def owner(db: Any) -> User: + return User.objects.create(username="owner", keycloak_id="owner") + + +@pytest.fixture +def dataset(owner: User) -> Dataset: + org = Organization.objects.create(name="Org", description="o", slug="org") + return Dataset.objects.create( + title="District health indicators", + slug="dhi", + description="d", + user=owner, + organization=org, + ) + + +@pytest.fixture +def add_resource(dataset: Dataset) -> Callable[..., Resource]: + """Create a resource holding a CSV file (fixture file name or raw bytes).""" + + def add(name: str, csv: Any, in_dataset: Optional[Dataset] = None) -> Resource: + content = fixture_bytes(csv) if isinstance(csv, str) else csv + resource = Resource.objects.create(name=name, dataset=in_dataset or dataset) + ResourceFileDetails.objects.create( + resource=resource, + file=ContentFile(content, name=f"{resource.id.hex}.csv"), + size=len(content), + format="csv", + ) + return resource + + return add diff --git a/tests/data_index/fixtures/districts_v1.csv b/tests/data_index/fixtures/districts_v1.csv new file mode 100644 index 00000000..1cb01ba4 --- /dev/null +++ b/tests/data_index/fixtures/districts_v1.csv @@ -0,0 +1,4 @@ +district,cases +Kamrup,10 +Nagaon,20 +Jorhat,30 diff --git a/tests/data_index/fixtures/districts_v2.csv b/tests/data_index/fixtures/districts_v2.csv new file mode 100644 index 00000000..e02f0ba8 --- /dev/null +++ b/tests/data_index/fixtures/districts_v2.csv @@ -0,0 +1,5 @@ +district,cases +Kamrup,11 +Nagaon,22 +Jorhat,33 +Dibrugarh,44 diff --git a/tests/data_index/fixtures/long_header.csv b/tests/data_index/fixtures/long_header.csv new file mode 100644 index 00000000..15305323 --- /dev/null +++ b/tests/data_index/fixtures/long_header.csv @@ -0,0 +1,3 @@ +district,Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anaemia among women aged 15 to 49 years who were tested during the survey round Prevalence of anae +Kamrup,1 +Nagaon,2 diff --git a/tests/data_index/fixtures/out_of_range.csv b/tests/data_index/fixtures/out_of_range.csv new file mode 100644 index 00000000..f8d6615a --- /dev/null +++ b/tests/data_index/fixtures/out_of_range.csv @@ -0,0 +1,3 @@ +district,count +Kamrup,1 +Nagaon,18446744073709551615 diff --git a/tests/data_index/fixtures/percent_header.csv b/tests/data_index/fixtures/percent_header.csv new file mode 100644 index 00000000..1b9e7886 --- /dev/null +++ b/tests/data_index/fixtures/percent_header.csv @@ -0,0 +1,4 @@ +district,Literacy rate (%),year +Kamrup,81.5,2021 +Nagaon,72.0,2021 +Jorhat,85.25,2022 diff --git a/tests/data_index/routers.py b/tests/data_index/routers.py new file mode 100644 index 00000000..56d4dcf7 --- /dev/null +++ b/tests/data_index/routers.py @@ -0,0 +1,18 @@ +"""Keep Django's own tables out of the test data database. + +In production the data database holds only data tables (one per tabular +upload). The router mirrors that in tests, so pytest-django creates no model +tables there and an orphan or "is this the main database?" check sees what +production would see. +""" + +from typing import Any, Optional + + +class DataDbTestRouter: + def allow_migrate( + self, db: str, app_label: str, model_name: Optional[str] = None, **hints: Any + ) -> Optional[bool]: + if db == "data_db": + return False + return None diff --git a/tests/data_index/support.py b/tests/data_index/support.py new file mode 100644 index 00000000..d9a04ef0 --- /dev/null +++ b/tests/data_index/support.py @@ -0,0 +1,60 @@ +"""Helpers shared by the data-index tests (fixtures live in conftest.py).""" + +from pathlib import Path +from typing import Any, List, Optional + +import pytest +from django.core.files.base import ContentFile +from django.db import connections + +from api.models import Resource + +FIXTURES = Path(__file__).parent / "fixtures" +DATA_DB = "data_db" + + +def uses_data_db(test: Any) -> Any: + """Mark a test that builds tables in the data database. + + It needs real commits (to see on_commit drops and the switch) and both + databases. Skipped without TEST_DATA_DB_URL; see conftest.py. + """ + test = pytest.mark.django_db(transaction=True, databases=["default", DATA_DB])(test) + return pytest.mark.data_db(test) + + +def list_public_tables() -> List[str]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + return sorted(row[0] for row in cursor.fetchall()) + + +def table_exists(name: str) -> bool: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT to_regclass(%s) IS NOT NULL", [f'public."{name}"']) + return bool(cursor.fetchone()[0]) + + +def table_rows(name: str, order_by: str = "1") -> List[tuple]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute(f'SELECT * FROM "{name}" ORDER BY {order_by}') + return cursor.fetchall() + + +def table_comment(name: str) -> Optional[str]: + with connections[DATA_DB].cursor() as cursor: + cursor.execute("SELECT obj_description(to_regclass(%s), 'pg_class')", [f'public."{name}"']) + return cursor.fetchone()[0] + + +def fixture_bytes(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def replace_file(resource: Resource, csv: Any) -> None: + """Save a new file for the resource, as an upload request does before indexing.""" + content = fixture_bytes(csv) if isinstance(csv, str) else csv + details = resource.resourcefiledetails + details.file = ContentFile(content, name=f"{resource.id.hex}.csv") + details.size = len(content) + details.save() diff --git a/tests/data_index/test_delete_cleanup.py b/tests/data_index/test_delete_cleanup.py new file mode 100644 index 00000000..fd19926c --- /dev/null +++ b/tests/data_index/test_delete_cleanup.py @@ -0,0 +1,181 @@ +"""Deleting a data-table row drops its table once the delete commits. + +Root cause this guards against: deleting a resource or a dataset never removed +its data tables, so the data database kept every table ever built. The one +receiver on the row's delete also covers re-uploads and admin deletes. +""" + +import os +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import psycopg2 +from django.db import connections, transaction +from structlog.testing import capture_logs + +from api.models import ResourceDataTable +from api.signals.data_table_signals import drop_table_after_row_delete +from api.utils.data_indexing import index_resource_data +from api.utils.data_tables import drop_data_table_if_unshared +from tests.data_index.support import uses_data_db + +REPO_ROOT = Path(__file__).resolve().parents[2] +SHARED_PREFIX = "district-wise-prevalence-of-anaemia-among-women-aged-15-49-dist" # 63 bytes + + +def create_legacy_table(name: str) -> None: + with connections["data_db"].cursor() as cursor: + cursor.execute(f'CREATE TABLE "{name}" (district text)') + + +@uses_data_db +def test_deleting_a_resource_drops_its_table(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + + resource.delete() + + assert not tables.exists(row.table_name) + + +@uses_data_db +def test_deleting_a_dataset_drops_every_table_of_its_resources( + add_resource: Any, dataset: Any, tables: Any +) -> None: + index_resource_data(add_resource("first.csv", "districts_v1.csv")) + index_resource_data(add_resource("second.csv", "districts_v2.csv")) + assert len(tables.names()) == 2 + + dataset.delete() + + assert tables.names() == [] + + +@uses_data_db +def test_a_rolled_back_delete_keeps_the_table(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + + try: + with transaction.atomic(): + resource.delete() + raise RuntimeError("the request failed after the delete") + except RuntimeError: + pass + + assert ResourceDataTable.objects.filter(pk=row.pk).exists() + assert tables.exists(row.table_name) + + +@uses_data_db +def test_a_legacy_table_two_rows_share_is_dropped_only_with_the_last_row( + add_resource: Any, tables: Any +) -> None: + first = ResourceDataTable.objects.create( + resource=add_resource("first.csv", "districts_v1.csv"), + table_name=SHARED_PREFIX + "rict-health-indicators-nfhs-4_1a2b3c4d", + ) + second = ResourceDataTable.objects.create( + resource=add_resource("second.csv", "districts_v1.csv"), + table_name=SHARED_PREFIX + "rict-health-indicators-nfhs-5_5e6f7a8b", + ) + create_legacy_table(first.table_name) + assert tables.names() == [SHARED_PREFIX] + + with capture_logs() as logs: + first.delete() + assert tables.names() == [SHARED_PREFIX] + assert any(log["event"] == "data_table_drop_skipped_shared" for log in logs) + + second.delete() + assert tables.names() == [] + + +@uses_data_db +def test_dropping_a_table_that_is_already_gone_does_not_fail_the_delete( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + ResourceDataTable.objects.create(resource=resource, table_name="rdt_never_built") + + resource.delete() + + assert not ResourceDataTable.objects.filter(resource_id=resource.pk).exists() + + +@uses_data_db +def test_a_drop_blocked_by_a_reader_is_logged_and_the_delete_still_succeeds( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = index_resource_data(resource) + settings = connections["data_db"].settings_dict + reader = psycopg2.connect( + dbname=settings["NAME"], + user=settings["USER"], + password=settings["PASSWORD"], + host=settings["HOST"], + port=settings["PORT"], + ) + try: + with reader.cursor() as cursor: + cursor.execute(f'LOCK TABLE "{row.table_name}" IN ACCESS SHARE MODE') + + with patch("api.utils.data_tables.DROP_LOCK_TIMEOUT", "200ms"), capture_logs() as logs: + resource.delete() + finally: + reader.rollback() + reader.close() + + assert not ResourceDataTable.objects.filter(pk=row.pk).exists() + assert tables.exists(row.table_name) + failures = [log for log in logs if log["event"] == "data_table_drop_failed"] + assert failures and failures[0]["table_name"] == row.table_name + + +def test_the_receiver_schedules_one_guarded_drop_per_deleted_row(add_resource: Any) -> None: + first = ResourceDataTable.objects.create(resource=add_resource("a.csv", "districts_v1.csv")) + second = ResourceDataTable.objects.create(resource=add_resource("b.csv", "districts_v1.csv")) + expected = [(first.table_name, first.pk), (second.table_name, second.pk)] + + with patch("api.signals.data_table_signals.transaction.on_commit") as on_commit: + ResourceDataTable.objects.filter(pk__in=[first.pk, second.pk]).delete() + + assert on_commit.call_count == 2 + scheduled = [] + for call in on_commit.call_args_list: + callback = call.args[0] + assert isinstance(callback, partial) + assert callback.func is drop_data_table_if_unshared + assert call.kwargs == {"using": "default", "robust": True} + scheduled.append(callback.args) + assert sorted(scheduled) == sorted(expected) + + +def test_the_receiver_is_connected_once_the_app_loads() -> None: + # A fresh interpreter: an import elsewhere in this test run can't connect it for us. + check = ( + "import django; django.setup()\n" + "from django.db.models.signals import post_delete\n" + "from api.models import ResourceDataTable\n" + "receivers = post_delete._live_receivers(ResourceDataTable)[0]\n" + "names = {(r.__module__, r.__name__) for r in receivers}\n" + "print(('api.signals.data_table_signals', 'drop_table_after_row_delete') in names)\n" + ) + env = {**os.environ, "DJANGO_SETTINGS_MODULE": "tests.test_settings"} + env.pop("TEST_DATA_DB_URL", None) + result = subprocess.run( + [sys.executable, "-c", check], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr[-2000:] + assert result.stdout.strip().splitlines()[-1] == "True" + assert drop_table_after_row_delete.__module__ == "api.signals.data_table_signals" diff --git a/tests/data_index/test_migrate_data_table_names.py b/tests/data_index/test_migrate_data_table_names.py new file mode 100644 index 00000000..03f776f2 --- /dev/null +++ b/tests/data_index/test_migrate_data_table_names.py @@ -0,0 +1,690 @@ +"""The one-off command that moves existing data tables to per-upload names. + +Root cause it cleans up after: tables used to be named from the resource name +and dataset title, cut by Postgres to 63 bytes, so two resources could share +one table (one of them serving the other's data). Short names can't have +collided and are renamed in place; long names may already hold another +resource's data and are rebuilt from each resource's own file. +""" + +import json +from datetime import datetime, timedelta +from datetime import timezone as dt_timezone +from io import StringIO +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pandas as pd +import psycopg2 +import pytest +from django.core.management import CommandError, call_command +from django.db import connections + +from api.models import AccessModel, AccessModelResource, ResourceDataTable +from api.utils.data_table_migration import ( + DataTableRow, + find_orphans, + rename_legacy_table, +) +from api.utils.data_table_names import data_table_name +from api.utils.data_tables import DataDbTable, build_data_table +from tests.data_index.support import uses_data_db + +SHORT_LEGACY = "casescsv_district-health-indicators_1a2b3c4d" +SHARED_PREFIX = "district-wise-prevalence-of-anaemia-among-women-aged-15-49-dist" # 63 bytes +LONG_FIRST = SHARED_PREFIX + "rict-health-indicators-nfhs-4_1a2b3c4d" +LONG_SECOND = SHARED_PREFIX + "rict-health-indicators-nfhs-5_5e6f7a8b" +WRONG_ROWS = [("Somewhere else", 999)] + + +def legacy_table(name: str, rows: List[tuple]) -> None: + """A table as the old code built it (Postgres cuts the name to 63 bytes).""" + with connections["data_db"].cursor() as cursor: + cursor.execute(f'CREATE TABLE "{name}" (district text, cases bigint)') + for row in rows: + cursor.execute(f'INSERT INTO "{name}" VALUES (%s, %s)', list(row)) + + +def legacy_row(resource: Any, name: str) -> ResourceDataTable: + return ResourceDataTable.objects.create(resource=resource, table_name=name) + + +def run_command(*args: str, report: Path) -> str: + out = StringIO() + call_command("migrate_data_table_names", *args, "--report", str(report), stdout=out) + return out.getvalue() + + +def read_report(path: Path) -> Dict[str, Any]: + return json.loads(path.read_text()) + + +def outcomes(report: Dict[str, Any]) -> Dict[str, str]: + return {entry["table_name"]: entry["outcome"] for entry in report["rows"]} + + +def current_table(resource: Any) -> str: + return ResourceDataTable.objects.get(resource=resource).table_name + + +def remove_stored_file(resource: Any) -> None: + details = resource.resourcefiledetails + details.file.storage.delete(details.file.name) + + +# --- Layer 1: the command against both databases ----------------------------- + + +@uses_data_db +def test_a_short_legacy_table_is_renamed_only_on_apply( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10), ("Nagaon", 20)]) + + run_command(report=tmp_path / "dry.json") + assert current_table(resource) == SHORT_LEGACY + assert tables.names() == [SHORT_LEGACY] + assert outcomes(read_report(tmp_path / "dry.json")) == {SHORT_LEGACY: "planned_rename"} + + run_command("--apply", report=tmp_path / "apply.json") + assert current_table(resource) == data_table_name(row.id) + assert tables.names() == [data_table_name(row.id)] + assert tables.rows(data_table_name(row.id)) == [("Kamrup", 10), ("Nagaon", 20)] + assert json.loads(tables.comment(data_table_name(row.id)))["resource_id"] == str(resource.id) + + +@uses_data_db +def test_a_table_two_rows_shared_is_rebuilt_from_each_resources_own_file( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) # the second upload overwrote the first's data + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == { + LONG_FIRST: "rebuilt", + LONG_SECOND: "rebuilt", + } + assert len(tables.rows(current_table(first))) == 3 + assert len(tables.rows(current_table(second))) == 4 + assert sorted(tables.names()) == sorted([current_table(first), current_table(second)]) + + +@uses_data_db +def test_a_long_legacy_table_with_one_row_left_is_rebuilt_not_renamed( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) # its former sharer, since deleted, overwrote it + + run_command("--apply", report=tmp_path / "apply.json") + + assert tables.rows(current_table(resource)) == [("Jorhat", 30), ("Kamrup", 10), ("Nagaon", 20)] + assert tables.names() == [current_table(resource)] + + +@uses_data_db +def test_a_long_legacy_table_whose_file_is_gone_is_renamed_and_marked_unverified( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + row = legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + remove_stored_file(resource) + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == {LONG_FIRST: "unverified"} + assert current_table(resource) == data_table_name(row.id) + assert tables.rows(data_table_name(row.id)) == WRONG_ROWS + assert json.loads(tables.comment(data_table_name(row.id)))["unverified"] is True + + # Every later run still lists it, though the row itself now looks migrated. + run_command(report=tmp_path / "later.json") + later = read_report(tmp_path / "later.json") + assert outcomes(later) == {data_table_name(row.id): "already_migrated"} + assert [t["resource_id"] for t in later["unverified_tables"]] == [str(resource.id)] + + +@uses_data_db +def test_a_sharing_row_whose_file_is_gone_stays_put_while_the_other_moves( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + remove_stored_file(first) + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == { + LONG_FIRST: "missing_file", + LONG_SECOND: "rebuilt", + } + assert current_table(first) == LONG_FIRST + assert SHARED_PREFIX in tables.names() # still used by the first row + assert len(tables.rows(current_table(second))) == 4 + + +@uses_data_db +def test_a_rerun_after_a_crash_between_rename_and_row_update_just_updates_the_row( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + with connections["data_db"].cursor() as cursor: + cursor.execute(f'ALTER TABLE "{SHORT_LEGACY}" RENAME TO "{data_table_name(row.id)}"') + + run_command("--apply", report=tmp_path / "apply.json") + + assert outcomes(read_report(tmp_path / "apply.json")) == {SHORT_LEGACY: "row_updated"} + assert current_table(resource) == data_table_name(row.id) + assert tables.rows(data_table_name(row.id)) == [("Kamrup", 10)] + + +@uses_data_db +def test_a_rename_blocked_by_a_reader_is_skipped_then_finished_on_a_rerun( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + settings = connections["data_db"].settings_dict + reader = psycopg2.connect( + dbname=settings["NAME"], + user=settings["USER"], + password=settings["PASSWORD"], + host=settings["HOST"], + port=settings["PORT"], + ) + try: + with reader.cursor() as cursor: + cursor.execute(f'LOCK TABLE "{SHORT_LEGACY}" IN ACCESS SHARE MODE') + with patch("api.utils.data_table_migration.RENAME_LOCK_TIMEOUT", "200ms"): + run_command("--apply", report=tmp_path / "blocked.json") + finally: + reader.rollback() + reader.close() + + assert outcomes(read_report(tmp_path / "blocked.json")) == {SHORT_LEGACY: "skipped_lock"} + assert current_table(resource) == SHORT_LEGACY + + run_command("--apply", report=tmp_path / "rerun.json") + assert current_table(resource) == data_table_name(row.id) + + +@uses_data_db +def test_orphans_are_listed_with_their_size_and_dropped_only_from_a_reviewed_report( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + owned = add_resource("cases.csv", "districts_v1.csv") + legacy_row(owned, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + legacy_table("forgotten-table_1", [("Kamrup", 1)]) + legacy_table("forgotten-table_2", [("Kamrup", 2)]) + old_build = (datetime.now(dt_timezone.utc) - timedelta(hours=2)).isoformat() + young_build = datetime.now(dt_timezone.utc).isoformat() + frame = pd.DataFrame({"a": [1]}) + build_data_table("rdt_old_orphan", frame, json.dumps({"built_at": old_build})) + build_data_table("rdt_in_flight", frame, json.dumps({"built_at": young_build})) + + run_command(report=tmp_path / "dry.json") + orphans = read_report(tmp_path / "dry.json")["orphans"] + assert sorted(o["name"] for o in orphans) == [ + "forgotten-table_1", + "forgotten-table_2", + "rdt_old_orphan", + ] + assert all(o["size_bytes"] > 0 for o in orphans) + + # Without a reviewed report nothing is dropped. + with pytest.raises(CommandError, match="--from-report"): + run_command("--apply", "--drop-orphans", report=tmp_path / "x.json") + assert "forgotten-table_1" in tables.names() + + # Since the review, a row started using forgotten-table_2, and a new orphan appeared. + late = add_resource("late.csv", "districts_v1.csv") + legacy_row(late, "forgotten-table_2") + legacy_table("not-reviewed", [("Kamrup", 3)]) + + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "dry.json"), + report=tmp_path / "drop.json", + ) + + assert sorted(read_report(tmp_path / "drop.json")["orphans_dropped"]) == [ + "forgotten-table_1", + "rdt_old_orphan", + ] + remaining = tables.names() + assert "forgotten-table_1" not in remaining and "rdt_old_orphan" not in remaining + # The late row's table was renamed by the same run, not dropped. + assert current_table(late).startswith("rdt_") + assert {current_table(late), "not-reviewed", "rdt_in_flight"} <= set(remaining) + + +@uses_data_db +def test_a_second_apply_after_completion_has_nothing_to_do( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + run_command("--apply", report=tmp_path / "first.json") + names_after_first = tables.names() + + output = run_command("--apply", report=tmp_path / "second.json") + + assert "nothing to do" in output + assert tables.names() == names_after_first + assert outcomes(read_report(tmp_path / "second.json")) == { + data_table_name(ResourceDataTable.objects.get(resource=resource).id): "already_migrated" + } + + +@uses_data_db +def test_rebuilds_list_the_access_models_whose_column_choices_reset( + add_resource: Any, dataset: Any, tmp_path: Path +) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + access_model = AccessModel.objects.create(name="Partners", dataset=dataset) + AccessModelResource.objects.create(access_model=access_model, resource=resource) + + run_command(report=tmp_path / "dry.json") + + assert read_report(tmp_path / "dry.json")["access_models_touched"] == [str(access_model.id)] + + +@uses_data_db +def test_the_command_refuses_when_the_data_database_is_the_main_database( + tmp_path: Path, +) -> None: + data = connections["data_db"].settings_dict + same = {"HOST": data["HOST"], "PORT": data["PORT"], "NAME": data["NAME"]} + with patch.dict(connections["default"].settings_dict, same): + with pytest.raises(CommandError, match="main database"): + run_command(report=tmp_path / "x.json") + assert not (tmp_path / "x.json").exists() + + +@uses_data_db +def test_the_command_refuses_when_the_data_database_holds_django_tables( + tables: Any, tmp_path: Path +) -> None: + with connections["data_db"].cursor() as cursor: + cursor.execute("CREATE TABLE django_migrations (id int)") + legacy_table("forgotten-table", [("Kamrup", 1)]) + + with pytest.raises(CommandError, match="django_migrations"): + run_command("--apply", report=tmp_path / "x.json") + assert "forgotten-table" in tables.names() + + +@uses_data_db +def test_a_dry_run_shows_what_apply_will_do_with_each_long_table( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + alone = add_resource("alone.csv", "districts_v1.csv") + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + lonely_name = "x" * 70 + legacy_row(alone, lonely_name) + legacy_table(lonely_name, WRONG_ROWS) + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + remove_stored_file(alone) + remove_stored_file(first) + + run_command(report=tmp_path / "dry.json") + + report = read_report(tmp_path / "dry.json") + assert outcomes(report) == { + lonely_name: "planned_unverified_rename", + LONG_FIRST: "planned_missing_file", + LONG_SECOND: "planned_rebuild", + } + assert {e["table_name"]: e["shared"] for e in report["rows"]} == { + lonely_name: False, + LONG_FIRST: True, + LONG_SECOND: True, + } + assert report["shared_groups"] == 1 + assert [e["table_name"] for e in report["needs_manual_fix"]] == [LONG_FIRST] + assert current_table(alone) == lonely_name # a dry run changes nothing + + +@uses_data_db +def test_a_long_table_whose_file_can_no_longer_become_a_table_is_kept_and_marked( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + alone = add_resource("alone.csv", "districts_v1.csv") + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + lonely_name = "x" * 70 + legacy_row(alone, lonely_name) + legacy_table(lonely_name, WRONG_ROWS) + legacy_row(first, LONG_FIRST) + legacy_row(second, LONG_SECOND) + legacy_table(LONG_SECOND, [("Kamrup", 11)]) + for resource in (alone, first): # later replaced by a PDF, which gets no table + resource.resourcefiledetails.format = "pdf" + resource.resourcefiledetails.save() + + run_command("--apply", report=tmp_path / "apply.json") + assert outcomes(read_report(tmp_path / "apply.json")) == { + lonely_name: "unverified", + LONG_FIRST: "not_indexable", # still shared at planning time: left alone + LONG_SECOND: "rebuilt", + } + + # With its sharer gone, the left-behind row is now alone, so a rerun marks it too. + run_command("--apply", report=tmp_path / "rerun.json") + assert outcomes(read_report(tmp_path / "rerun.json"))[LONG_FIRST] == "unverified" + + output = run_command("--apply", report=tmp_path / "last.json") + assert "nothing to do" in output + assert len(read_report(tmp_path / "last.json")["unverified_tables"]) == 2 + + +@uses_data_db +def test_rebuilds_read_files_with_no_time_limit(add_resource: Any, tmp_path: Path) -> None: + resource = add_resource("first.csv", "districts_v1.csv") + legacy_row(resource, LONG_FIRST) + legacy_table(LONG_FIRST, WRONG_ROWS) + + with patch( + "api.utils.data_table_migration.index_resource_data", return_value=MagicMock() + ) as index: + run_command("--apply", report=tmp_path / "apply.json") + + index.assert_called_once() + assert index.call_args.kwargs == {"load_timeout": None} + + +@uses_data_db +def test_a_row_whose_table_is_gone_is_reindexed_or_left_for_a_person( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + has_file = add_resource("has-file.csv", "districts_v1.csv") + no_file = add_resource("no-file.csv", "districts_v1.csv") + gone = ResourceDataTable.objects.create(resource=has_file) # rdt_ name, table never built + ResourceDataTable.objects.create(resource=no_file) + remove_stored_file(no_file) + + output = run_command("--apply", report=tmp_path / "apply.json") + + report = read_report(tmp_path / "apply.json") + assert sorted(e["outcome"] for e in report["rows"]) == ["missing_file", "reindexed"] + assert current_table(has_file) != gone.table_name + assert len(tables.rows(current_table(has_file))) == 3 + assert "nothing to do" not in output + assert "nothing to do" in run_command("--apply", report=tmp_path / "rerun.json") + + +@uses_data_db +def test_resource_id_limits_the_run_to_that_resource( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + target = add_resource("target.csv", "districts_v1.csv") + other = add_resource("other.csv", "districts_v1.csv") + legacy_row(target, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + legacy_row(other, "other-legacy-table") + legacy_table("other-legacy-table", [("Kamrup", 1)]) + + run_command("--apply", "--resource-id", str(target.id), report=tmp_path / "one.json") + + assert current_table(target).startswith("rdt_") + assert current_table(other) == "other-legacy-table" + + +@uses_data_db +def test_a_rename_backs_off_when_an_upload_replaced_the_row_after_planning( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + planned = DataTableRow(row.id, resource.id, SHORT_LEGACY, row.created) + ResourceDataTable.objects.filter(pk=row.pk).update(table_name="rdt_" + "f" * 32) + + assert rename_legacy_table(planned) == "changed" + assert SHORT_LEGACY in tables.names() + + +@uses_data_db +def test_a_bad_reviewed_report_stops_the_run_before_any_change( + add_resource: Any, tables: Any, tmp_path: Path +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + legacy_row(resource, SHORT_LEGACY) + legacy_table(SHORT_LEGACY, [("Kamrup", 10)]) + (tmp_path / "edited.json").write_text('{"orphans": [{"name": "a"},]}') # trailing comma + + for reviewed in ("missing.json", "edited.json"): + with pytest.raises(CommandError, match="reviewed report"): + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / reviewed), + report=tmp_path / "out.json", + ) + + assert current_table(resource) == SHORT_LEGACY + + +@uses_data_db +def test_the_report_may_not_overwrite_the_reviewed_report(tmp_path: Path) -> None: + run_command(report=tmp_path / "dry.json") + reviewed = (tmp_path / "dry.json").read_text() + + with pytest.raises(CommandError, match="overwrite"): + run_command( + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "dry.json"), + report=tmp_path / "dry.json", + ) + + assert (tmp_path / "dry.json").read_text() == reviewed + + +# --- Layer 2: orphan rules --------------------------------------------------- + +NOW = datetime(2026, 11, 13, 12, 0, tzinfo=dt_timezone.utc) + + +def table(name: str, built_minutes_ago: Any = None) -> DataDbTable: + comment = None + if built_minutes_ago is not None: + built_at = (NOW - timedelta(minutes=built_minutes_ago)).isoformat() + comment = json.dumps({"built_at": built_at}) + return DataDbTable(name=name, size_bytes=8192, comment=comment) + + +def test_orphans_never_include_django_tables_rows_tables_or_fresh_builds() -> None: + row_id = __import__("uuid").uuid4() + rows = [ + DataTableRow(row_id=row_id, resource_id=row_id, table_name=LONG_FIRST, created=NOW), + ] + candidates = [ + table("resource_data_table"), # a Django model's table + table("django_migrations"), + table(SHARED_PREFIX), # the row's (cut) table + table(data_table_name(row_id)), # the row's renamed table, row not yet updated + table("rdt_" + "a" * 32, built_minutes_ago=5), # an upload still switching + table("rdt_" + "b" * 32, built_minutes_ago=90), + table("rdt_" + "c" * 32), # no readable build time + table("forgotten-table"), + ] + + names = [t.name for t in find_orphans(rows, candidates, now=NOW)] + + assert names == ["rdt_" + "b" * 32, "rdt_" + "c" * 32, "forgotten-table"] + + +# --- Layer 3: the command's flow (helpers mocked) ---------------------------- + +COMMAND = "api.management.commands.migrate_data_table_names" + + +@pytest.fixture +def command_helpers() -> Any: + parent = MagicMock(unsafe=True) # one helper's name starts with "assert" + names = [ + "assert_data_db_is_separate", + "read_data_table_rows", + "list_data_db_tables", + "plan_table_migration", + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "rebuild_decision", + "reindex_decision", + "reindex_missing_table", + "find_unverified", + "access_models_touched", + "read_reviewed_orphans", + "find_orphans", + "drop_reviewed_orphans", + "write_report", + ] + patchers = [patch(f"{COMMAND}.{name}", getattr(parent, name)) for name in names] + for p in patchers: + p.start() + row = lambda name: DataTableRow( # noqa: E731 + row_id=name, resource_id=name, table_name=name, created=NOW + ) + plan = MagicMock() + plan.already_migrated = [] + plan.row_updates = [row("crashed")] + plan.renames = [row("short")] + plan.rebuilds = [(row("long_old"), False), (row("long_new"), True)] + plan.missing_tables = [row("missing")] + plan.orphans = [] + plan.shared_groups = 1 + parent.plan_table_migration.return_value = plan + parent.point_row_at_its_table.return_value = "row_updated" + parent.rename_legacy_table.return_value = "renamed" + parent.rebuild_from_file.return_value = "rebuilt" + parent.reindex_missing_table.return_value = "reindexed" + parent.rebuild_decision.return_value = "rebuild" + parent.reindex_decision.return_value = "reindex" + parent.find_unverified.return_value = [] + parent.access_models_touched.return_value = [] + parent.drop_reviewed_orphans.return_value = [] + yield parent + for p in patchers: + p.stop() + + +def top_level_calls(parent: MagicMock) -> List[str]: + return [c[0] for c in parent.mock_calls if c[0] and "." not in c[0] and "(" not in c[0]] + + +def test_a_dry_run_checks_safety_first_and_changes_nothing( + command_helpers: MagicMock, tmp_path: Path +) -> None: + call_command( + "migrate_data_table_names", "--report", str(tmp_path / "r.json"), stdout=StringIO() + ) + + calls = top_level_calls(command_helpers) + assert calls[0] == "assert_data_db_is_separate" + assert "write_report" in calls + for action in [ + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "reindex_missing_table", + "drop_reviewed_orphans", + ]: + assert action not in calls + + +def test_apply_fixes_crashed_rows_then_renames_then_rebuilds_in_order( + command_helpers: MagicMock, tmp_path: Path +) -> None: + call_command( + "migrate_data_table_names", + "--apply", + "--report", + str(tmp_path / "r.json"), + stdout=StringIO(), + ) + + actions = [ + (c[0], c.args[0].table_name) + for c in command_helpers.mock_calls + if c[0] + in { + "point_row_at_its_table", + "rename_legacy_table", + "rebuild_from_file", + "reindex_missing_table", + } + ] + assert actions == [ + ("point_row_at_its_table", "crashed"), + ("rename_legacy_table", "short"), + ("rebuild_from_file", "long_old"), + ("rebuild_from_file", "long_new"), + ("reindex_missing_table", "missing"), + ] + command_helpers.drop_reviewed_orphans.assert_not_called() + + +def test_the_command_stops_before_reading_anything_when_safety_fails( + command_helpers: MagicMock, tmp_path: Path +) -> None: + from api.utils.data_tables import DataDbNotSeparate + + command_helpers.assert_data_db_is_separate.side_effect = DataDbNotSeparate("same database") + + with pytest.raises(CommandError): + call_command("migrate_data_table_names", "--apply", "--report", str(tmp_path / "r.json")) + + assert top_level_calls(command_helpers) == ["assert_data_db_is_separate"] + + +def test_orphans_are_dropped_only_with_apply_and_a_reviewed_report( + command_helpers: MagicMock, tmp_path: Path +) -> None: + with pytest.raises(CommandError): + call_command("migrate_data_table_names", "--drop-orphans", "--from-report", "r.json") + command_helpers.drop_reviewed_orphans.assert_not_called() + + command_helpers.read_reviewed_orphans.return_value = {"forgotten-table"} + call_command( + "migrate_data_table_names", + "--apply", + "--drop-orphans", + "--from-report", + str(tmp_path / "reviewed.json"), + "--report", + str(tmp_path / "r.json"), + stdout=StringIO(), + ) + command_helpers.read_reviewed_orphans.assert_called_once_with(str(tmp_path / "reviewed.json")) + command_helpers.drop_reviewed_orphans.assert_called_once_with( + {"forgotten-table"}, command_helpers.find_orphans.return_value + ) diff --git a/tests/data_index/test_reader_regression.py b/tests/data_index/test_reader_regression.py new file mode 100644 index 00000000..152a8b4a --- /dev/null +++ b/tests/data_index/test_reader_regression.py @@ -0,0 +1,59 @@ +"""Every reader of data tables still works on the new per-upload tables. + +The readers look the table up from the resource's ResourceDataTable row, so +the new names should be invisible to them. This proves it for each reader, +on a header with a percent sign (common in indicator tables). +""" + +from typing import Any + +from api.models import ResourcePreviewDetails, ResourceSchema +from api.utils.data_indexing import ( + fetch_resource_data, + get_preview_data, + get_resource_columns, + get_row_count, + index_resource_data, + query_resource_data, +) +from tests.data_index.support import uses_data_db + +COLUMNS = ["district", "Literacy rate (%)", "year"] + + +@uses_data_db +def test_every_reader_works_on_a_new_table_with_a_percent_header(add_resource: Any) -> None: + resource = add_resource("literacy.csv", "percent_header.csv") + resource.preview_enabled = True + resource.preview_details = ResourcePreviewDetails.objects.create(is_all_entries=True) + resource.save() + row = index_resource_data(resource) + assert row is not None and row.table_name.startswith("rdt_") + + assert get_row_count(resource) == 3 + + preview = get_preview_data(resource) + assert preview is not None + assert preview.columns == COLUMNS + assert len(preview.rows) == 3 + + assert get_resource_columns(resource) == COLUMNS + + fetched = fetch_resource_data( + resource, + filters={"Literacy rate (%)__gte": 80}, + columns=["district", "Literacy rate (%)"], + order_by=["-Literacy rate (%)"], + ) + assert fetched["rows"] == [["Jorhat", 85.25], ["Kamrup", 81.5]] + assert fetched["total"] == 2 + + chart = query_resource_data( + resource, 'SELECT district, "Literacy rate (%)" FROM {{table}} WHERE year = 2022' + ) + assert chart is not None + assert chart.values.tolist() == [["Jorhat", 85.25]] + + # With no column rows, the column list falls back to the table itself. + ResourceSchema.objects.filter(resource=resource).delete() + assert get_resource_columns(resource) == COLUMNS diff --git a/tests/data_index/test_reindex_switch.py b/tests/data_index/test_reindex_switch.py new file mode 100644 index 00000000..db6ef8e0 --- /dev/null +++ b/tests/data_index/test_reindex_switch.py @@ -0,0 +1,476 @@ +"""Re-uploads build the new table first and switch over in one commit. + +Root causes this guards against: +- The old table was dropped before the new one existed, so readers saw empty or + half-written data, and a failed re-upload lost the data for good. +- The column-row rebuild swallowed errors per column. Inside the switch's + transaction one failure silently rolled everything back while the upload + still reported success. +- Two overlapping uploads of one resource each indexed their own file, and the + slower (older) one could switch in last, so the download and the served rows + disagreed. +""" + +from datetime import datetime, timezone +from typing import Any, List +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +from django.db import connections, transaction +from django.db.utils import ProgrammingError +from rest_framework.test import APIClient +from structlog.testing import capture_logs + +from api.models import Resource, ResourceDataTable, ResourceSchema +from api.utils import data_tables +from api.utils.data_indexing import index_resource_data +from api.utils.data_tables import ( + FileIdentity, + LoadedFile, + build_data_table, + drop_unswitched_table, + has_indexable_file, + load_resource_frame, + rebuild_resource_schema, +) +from tests.data_index.support import fixture_bytes, replace_file, uses_data_db + +# --- Layer 1: the real flow against both databases -------------------------- + + +@uses_data_db +def test_first_upload_creates_a_named_table_with_its_columns( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + + row = index_resource_data(resource) + + assert row is not None and row.table_name == "rdt_" + row.id.hex + assert ResourceDataTable.objects.get(resource=resource).table_name == row.table_name + assert tables.rows(row.table_name) == [("Jorhat", 30), ("Kamrup", 10), ("Nagaon", 20)] + assert list( + ResourceSchema.objects.filter(resource=resource) + .order_by("id") + .values_list("field_name", "format") + ) == [("district", "STRING"), ("cases", "INTEGER")] + assert str(resource.id) in tables.comment(row.table_name) + assert "cases.csv" in tables.comment(row.table_name) + + +@uses_data_db +def test_reupload_switches_to_a_new_table_and_drops_the_old_one( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + ResourceSchema.objects.filter(resource=resource, field_name="cases").update( + description="Confirmed cases" + ) + + replace_file(resource, "districts_v2.csv") + new = index_resource_data(resource) + + assert new is not None and new.table_name != old.table_name + assert ResourceDataTable.objects.get(resource=resource).pk == new.pk + assert len(tables.rows(new.table_name)) == 4 + assert not tables.exists(old.table_name) + assert tables.names() == [new.table_name] + assert ( + ResourceSchema.objects.get(resource=resource, field_name="cases").description + == "Confirmed cases" + ) + + +@uses_data_db +def test_readers_see_the_complete_old_table_until_the_switch_commits( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + seen: dict = {} + real_lock = data_tables.lock_resource + real_drop = data_tables.drop_data_table_if_unshared + + def spy_lock(res: Resource) -> None: + new_tables = [t for t in tables.names() if t != old.table_name] + seen["new_rows_at_switch"] = len(tables.rows(new_tables[0])) + seen["old_exists_at_switch"] = tables.exists(old.table_name) + # The lock only holds inside the switch's transaction (Postgres refuses it outside). + seen["lock_inside_transaction"] = transaction.get_connection().in_atomic_block + real_lock(res) + + def spy_drop(table_name: str, row_id: Any) -> None: + seen["dropped"] = table_name + seen["open_transaction_at_drop"] = transaction.get_connection().in_atomic_block + seen["new_row_saved_at_drop"] = ( + ResourceDataTable.objects.get(resource=resource).table_name != old.table_name + ) + real_drop(table_name, row_id) + + with ( + patch("api.utils.data_indexing.lock_resource", spy_lock), + patch("api.signals.data_table_signals.drop_data_table_if_unshared", spy_drop), + ): + new = index_resource_data(resource) + + assert new is not None + assert seen == { + "new_rows_at_switch": 4, + "old_exists_at_switch": True, + "lock_inside_transaction": True, + "dropped": old.table_name, + "open_transaction_at_drop": False, + "new_row_saved_at_drop": True, + } + + +@uses_data_db +def test_a_failed_build_keeps_the_old_data_and_leaves_no_table( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + + # A count past the BIGINT range: the copy into the new table fails midway. + replace_file(resource, "out_of_range.csv") + with capture_logs() as logs: + assert index_resource_data(resource) is None + + assert ResourceDataTable.objects.get(resource=resource).pk == old.pk + assert tables.names() == [old.table_name] + assert len(tables.rows(old.table_name)) == 3 + # The failure is logged by its code, never by the offending cell's value. + failures = [log for log in logs if log["event"] == "data_index_failed"] + assert failures and failures[0]["error_code"] == "22003" + assert failures[0]["where"].startswith("data_tables.py:") + assert "18446744073709551615" not in repr(logs) + + +@uses_data_db +def test_a_header_too_long_for_the_column_list_fails_the_switch_and_keeps_the_old_data( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + old_columns = list( + ResourceSchema.objects.filter(resource=resource).values_list("field_name", flat=True) + ) + + replace_file(resource, "long_header.csv") + assert index_resource_data(resource) is None + + assert ResourceDataTable.objects.get(resource=resource).pk == old.pk + assert tables.names() == [old.table_name] + assert ( + list(ResourceSchema.objects.filter(resource=resource).values_list("field_name", flat=True)) + == old_columns + ) + + +@uses_data_db +def test_a_reader_still_holding_the_old_name_gets_does_not_exist_not_new_rows( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + stale = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + index_resource_data(resource) + + with pytest.raises(ProgrammingError, match="does not exist"): + with connections["data_db"].cursor() as cursor: + cursor.execute(f'SELECT * FROM "{stale.table_name}"') + + +@uses_data_db +def test_a_run_stopped_while_the_old_table_is_dropped_keeps_the_new_table( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + old = index_resource_data(resource) + replace_file(resource, "districts_v2.csv") + + def stopped_by_the_operator(table_name: str, row_id: Any) -> None: + raise KeyboardInterrupt # Ctrl+C while the old table's drop waits for a reader + + with patch( + "api.signals.data_table_signals.drop_data_table_if_unshared", stopped_by_the_operator + ): + with pytest.raises(KeyboardInterrupt): + index_resource_data(resource) + + current = ResourceDataTable.objects.get(resource=resource) + assert current.pk != old.pk # the switch had committed + assert tables.exists(current.table_name) + assert len(tables.rows(current.table_name)) == 4 + + +@uses_data_db +def test_dropping_an_unswitched_build_checks_the_row_first(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + saved = index_resource_data(resource) + unsaved = data_tables.new_data_table_row(resource) + build_data_table(unsaved.table_name, pd.DataFrame({"a": [1]}), "{}") + + drop_unswitched_table(saved) + drop_unswitched_table(unsaved) + assert tables.exists(saved.table_name) + assert not tables.exists(unsaved.table_name) + + with patch("api.utils.data_tables.data_table_row_exists", side_effect=RuntimeError("db gone")): + drop_unswitched_table(saved) # can't tell: leave it for the orphan audit + assert tables.exists(saved.table_name) + + +@uses_data_db +def test_a_slower_build_of_an_older_file_backs_off_when_a_newer_file_was_saved( + add_resource: Any, tables: Any +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + real_build = data_tables.build_data_table + newer: dict = {} + + def build_then_newer_upload_lands(table_name: str, frame: pd.DataFrame, comment: str) -> None: + real_build(table_name, frame, comment) + if newer: + return # the other request's own build + newer["started"] = True + # Meanwhile another request saves a newer file and finishes indexing it. + other_request = Resource.objects.get(pk=resource.pk) + replace_file(other_request, "districts_v2.csv") + newer["row"] = index_resource_data(other_request) + + with patch("api.utils.data_indexing.build_data_table", build_then_newer_upload_lands): + assert index_resource_data(resource) is None + + current = ResourceDataTable.objects.get(resource=resource) + assert current.pk == newer["row"].pk + assert tables.names() == [current.table_name] + assert len(tables.rows(current.table_name)) == 4 + + +@uses_data_db +def test_indexing_inside_an_outer_transaction_is_refused(add_resource: Any, tables: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + + with capture_logs() as logs, transaction.atomic(): + assert index_resource_data(resource) is None + + assert tables.names() == [] + assert not ResourceDataTable.objects.filter(resource=resource).exists() + assert any(log["event"] == "data_index_refused_inside_transaction" for log in logs) + + +@uses_data_db +def test_reupload_keeps_the_datasets_default_table( + add_resource: Any, owner: Any, dataset: Any +) -> None: + first = add_resource("first.csv", "districts_v1.csv") + second = add_resource("second.csv", "districts_v2.csv") + first_row = index_resource_data(first) + index_resource_data(second) + + replace_file(first, "districts_v2.csv") + replaced = index_resource_data(first) + + assert replaced is not None and replaced.created == first_row.created + assert ResourceDataTable.objects.get(pk=replaced.pk).created == first_row.created + client = APIClient() + client.force_authenticate(owner) + response = client.get(f"/api/datasets/{dataset.id}/data/") + assert response.status_code == 200, response.content + assert response.json()["resource_id"] == str(first.id) + + +# --- Layer 1: helpers on the main database only ------------------------------ + + +def test_a_resource_without_a_supported_file_is_not_indexable(add_resource: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + assert has_indexable_file(resource) + + resource.resourcefiledetails.format = "pdf" + assert not has_indexable_file(resource) + + bare = Resource.objects.create(name="empty", dataset=resource.dataset) + assert not has_indexable_file(bare) + + +def test_load_remembers_which_stored_file_it_read(add_resource: Any) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + details = resource.resourcefiledetails + + loaded = load_resource_frame(resource) + + assert loaded is not None + assert loaded.file == FileIdentity(details.file.name, details.sha256) + assert list(loaded.frame.columns) == ["district", "cases"] + + +def test_load_returns_nothing_for_a_file_with_only_a_header(add_resource: Any) -> None: + resource = add_resource("cases.csv", b"district,cases\n") + assert load_resource_frame(resource) is None + + +def test_load_cleans_quotes_line_breaks_and_duplicate_headers(add_resource: Any) -> None: + # pandas renames plain duplicates itself; these two only collide once the quote goes. + resource = add_resource("cases.csv", b'"dis""trict","a\nb","x""",x\nKamrup,1,2,3\n') + loaded = load_resource_frame(resource) + assert loaded is not None + assert list(loaded.frame.columns) == ["district", "a b", "x", "x_1"] + + +def test_rebuild_columns_lets_a_database_error_escape(add_resource: Any) -> None: + from django.db import DatabaseError + + resource = add_resource("cases.csv", "districts_v1.csv") + frame = pd.DataFrame({"district": ["Kamrup"], "cases": [1]}) + + def failing_create(**kwargs: Any) -> None: + raise DatabaseError("value too long for type character varying(255)") + + with patch.object(ResourceSchema.objects, "create", failing_create): + with pytest.raises(DatabaseError): + rebuild_resource_schema(resource, frame) + + +def test_rebuild_columns_refuses_a_header_longer_than_the_column_list_allows( + add_resource: Any, +) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + frame = pd.DataFrame({"district": ["Kamrup"], "x" * 256: [1]}) + with pytest.raises(ValueError, match="column 2"): + rebuild_resource_schema(resource, frame) + assert not ResourceSchema.objects.filter(resource=resource).exists() + + +# --- Layer 3: the flow's order and clean-up (helpers mocked) ----------------- + +FLOW_HELPERS = [ + "is_inside_transaction", + "has_indexable_file", + "load_resource_frame", + "new_data_table_row", + "describe_data_table", + "build_data_table", + "lock_resource", + "stored_file_identity", + "retire_data_table_row", + "save_data_table_row", + "rebuild_resource_schema", + "data_table_row_exists", + "drop_unswitched_table", +] +FILE = FileIdentity("resources/cases.csv", "abc123") + + +@pytest.fixture +def flow() -> Any: + """Every helper the flow calls, mocked on one parent so call order is recorded.""" + parent = MagicMock() + patchers = [ + patch(f"api.utils.data_indexing.{name}", getattr(parent, name)) for name in FLOW_HELPERS + ] + for p in patchers: + p.start() + parent.is_inside_transaction.return_value = False + parent.has_indexable_file.return_value = True + parent.load_resource_frame.return_value = LoadedFile(pd.DataFrame({"a": [1]}), FILE) + parent.new_data_table_row.return_value = MagicMock(table_name="rdt_new", pk="new-id") + parent.stored_file_identity.return_value = FILE + parent.retire_data_table_row.return_value = datetime(2026, 1, 1, tzinfo=timezone.utc) + parent.data_table_row_exists.return_value = True + yield parent + for p in patchers: + p.stop() + + +def called(flow: Any) -> List[str]: + return [c[0] for c in flow.mock_calls if "." not in c[0]] + + +def test_flow_builds_then_locks_before_retiring_the_old_row(flow: Any) -> None: + row = index_resource_data(MagicMock()) + + assert row is flow.new_data_table_row.return_value + assert called(flow) == [ + "is_inside_transaction", + "has_indexable_file", + "load_resource_frame", + "new_data_table_row", + "describe_data_table", + "build_data_table", + "lock_resource", + "stored_file_identity", + "retire_data_table_row", + "save_data_table_row", + "rebuild_resource_schema", + "data_table_row_exists", + ] + flow.save_data_table_row.assert_called_once_with( + flow.new_data_table_row.return_value, created=flow.retire_data_table_row.return_value + ) + + +def raise_error(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("boom") + + +@pytest.mark.parametrize( + "helper", + ["build_data_table", "lock_resource", "retire_data_table_row", "rebuild_resource_schema"], +) +def test_flow_drops_its_table_when_a_step_fails(flow: Any, helper: str) -> None: + getattr(flow, helper).side_effect = raise_error + + assert index_resource_data(MagicMock()) is None + + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) + + +def test_flow_drops_its_table_when_a_newer_file_was_saved(flow: Any) -> None: + flow.stored_file_identity.return_value = FileIdentity("resources/newer.csv", "def456") + + assert index_resource_data(MagicMock()) is None + + flow.retire_data_table_row.assert_not_called() + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) + + +def test_flow_drops_its_table_when_the_new_row_is_missing_after_the_commit(flow: Any) -> None: + flow.data_table_row_exists.return_value = False + + assert index_resource_data(MagicMock()) is None + + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) + + +def test_flow_drops_its_table_even_when_interrupted(flow: Any) -> None: + flow.build_data_table.side_effect = KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + index_resource_data(MagicMock()) + + flow.drop_unswitched_table.assert_called_once_with(flow.new_data_table_row.return_value) + + +def test_flow_keeps_the_table_after_a_confirmed_switch(flow: Any) -> None: + index_resource_data(MagicMock()) + flow.drop_unswitched_table.assert_not_called() + + +@pytest.mark.parametrize("stop_at", ["inside_transaction", "no_file", "nothing_loaded"]) +def test_flow_builds_nothing_when_there_is_nothing_to_index(flow: Any, stop_at: str) -> None: + if stop_at == "inside_transaction": + flow.is_inside_transaction.return_value = True + elif stop_at == "no_file": + flow.has_indexable_file.return_value = False + else: + flow.load_resource_frame.return_value = None + + assert index_resource_data(MagicMock()) is None + + flow.build_data_table.assert_not_called() + flow.drop_unswitched_table.assert_not_called() diff --git a/tests/data_index/test_table_names.py b/tests/data_index/test_table_names.py new file mode 100644 index 00000000..d2c98ceb --- /dev/null +++ b/tests/data_index/test_table_names.py @@ -0,0 +1,93 @@ +"""Every tabular upload gets its own table, named from its own row id. + +Root cause this guards against: table names used to be built from the resource +name and dataset title (up to 110 characters). Postgres keeps only the first 63 +bytes of a name, so two resources with long, similar names shared one table and +a re-upload of one replaced the other's data. +""" + +import json +import uuid +from datetime import datetime, timezone + +from api.models import ResourceDataTable +from api.utils.data_indexing import index_resource_data +from api.utils.data_table_names import data_table_name, physical_table_name +from api.utils.data_tables import build_data_table, describe_data_table +from tests.data_index.support import uses_data_db + +LEGACY_NAME = ( + "district-wise-prevalence-of-anaemia-among-women-aged-15-49-" + "district-health-indicators-assam-nfhs-5_1a2b3c4d" +) + + +def test_a_table_name_is_rdt_and_the_row_id_in_lowercase_hex() -> None: + row_id = uuid.UUID("0123456789ABCDEF0123456789ABCDEF") + assert data_table_name(row_id) == "rdt_0123456789abcdef0123456789abcdef" + assert len(data_table_name(row_id)) == 36 + + +def test_physical_name_is_cut_to_63_bytes_like_postgres() -> None: + assert len(LEGACY_NAME) > 100 + assert physical_table_name(LEGACY_NAME) == LEGACY_NAME[:63] + short = data_table_name(uuid.uuid4()) + assert physical_table_name(short) == short + + +def test_physical_name_never_cuts_a_character_in_half() -> None: + # 62 ASCII bytes then a 2-byte character: Postgres keeps 62 bytes, not 63. + name = "a" * 62 + "é" + "tail" + assert physical_table_name(name) == "a" * 62 + + +def test_a_new_row_is_named_from_its_own_id(add_resource: object) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = ResourceDataTable.objects.create(resource=resource) + assert row.table_name == "rdt_" + row.id.hex + + +def test_an_existing_row_keeps_its_stored_name_on_save(add_resource: object) -> None: + resource = add_resource("cases.csv", "districts_v1.csv") + row = ResourceDataTable.objects.create(resource=resource, table_name=LEGACY_NAME) + row.save() + row.refresh_from_db() + assert row.table_name == LEGACY_NAME + + +@uses_data_db +def test_two_resources_with_long_similar_names_get_their_own_tables( + add_resource: object, tables: object +) -> None: + prefix = "District wise prevalence of anaemia among women aged 15 to 49 " + first = add_resource(prefix + "(NFHS-4)", "districts_v1.csv") + second = add_resource(prefix + "(NFHS-5)", "districts_v2.csv") + + first_row = index_resource_data(first) + second_row = index_resource_data(second) + + assert first_row is not None and second_row is not None + assert first_row.table_name != second_row.table_name + assert [r[1] for r in tables.rows(first_row.table_name)] == [30, 10, 20] + assert len(tables.rows(second_row.table_name)) == 4 + + +@uses_data_db +def test_the_table_comment_is_stored_as_text_not_run_as_sql( + add_resource: object, tables: object +) -> None: + import pandas as pd + + resource = add_resource("cases'; DROP TABLE x; --\"", "districts_v1.csv") + built_at = datetime(2026, 11, 13, 10, 30, tzinfo=timezone.utc) + comment = describe_data_table(resource, built_at=built_at) + + build_data_table("rdt_comment_check", pd.DataFrame({"a": [1]}), comment) + + stored = json.loads(tables.comment("rdt_comment_check")) + assert stored == { + "resource_id": str(resource.id), + "resource": "cases'; DROP TABLE x; --\"", + "dataset": "District health indicators", + "built_at": "2026-11-13T10:30:00+00:00", + } diff --git a/tests/data_index/test_upload_mutations.py b/tests/data_index/test_upload_mutations.py new file mode 100644 index 00000000..c5c9f313 --- /dev/null +++ b/tests/data_index/test_upload_mutations.py @@ -0,0 +1,128 @@ +"""Upload, replace and delete through the GraphQL API, read through the data API. + +Who may call these mutations is unchanged and covered by +tests/schema/test_resource_permissions.py; these tests follow a dataset owner +through the data-table lifecycle end to end. +""" + +import types +from typing import Any, Dict + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory +from rest_framework.test import APIClient + +from api.models import Resource, ResourceDataTable +from api.schema.schema import schema +from tests.data_index.support import fixture_bytes, uses_data_db + +CREATE = """mutation ($ds: UUID!, $file: Upload!) { + createFileResources(fileResourceInput: {dataset: $ds, files: [$file]}) { id } +}""" +REPLACE = """mutation ($id: UUID!, $file: Upload!) { + updateFileResource(fileResourceInput: {id: $id, file: $file}) { + __typename ... on OperationInfo { messages { message } } + } +}""" +RESET = """mutation ($id: UUID!) { + resetFileResourceSchema(resourceId: $id) { + __typename ... on OperationInfo { messages { message } } + } +}""" +DELETE_RESOURCE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" +DELETE_DATASET = "mutation ($id: UUID!) { deleteDataset(datasetId: $id) }" + + +def run(user: Any, query: str, **variables: Any) -> Dict[str, Any]: + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + result = schema.execute_sync(query, variable_values=variables, context_value=ctx) + assert result.errors is None, result.errors + payload = next(iter(result.data.values())) + if isinstance(payload, dict): + assert payload.get("__typename") != "OperationInfo", payload + return result.data + + +def csv_upload(name: str) -> SimpleUploadedFile: + return SimpleUploadedFile("cases.csv", fixture_bytes(name), content_type="text/csv") + + +def data_api(owner: Any, resource_id: Any) -> Any: + client = APIClient() + client.force_authenticate(owner) + return client.get(f"/api/resources/{resource_id}/data/?order_by=district") + + +def upload(owner: Any, dataset: Any, name: str) -> Resource: + data = run(owner, CREATE, ds=str(dataset.id), file=csv_upload(name)) + return Resource.objects.get(id=data["createFileResources"][0]["id"]) + + +@uses_data_db +def test_an_uploaded_csv_is_served_from_its_own_table( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + + table = ResourceDataTable.objects.get(resource=resource).table_name + assert table.startswith("rdt_") and tables.exists(table) + response = data_api(owner, resource.id) + assert response.status_code == 200, response.content + assert response.json()["rows"] == [["Jorhat", 30], ["Kamrup", 10], ["Nagaon", 20]] + + +@uses_data_db +def test_a_replaced_file_is_served_and_its_old_table_is_gone( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + old_table = ResourceDataTable.objects.get(resource=resource).table_name + + run(owner, REPLACE, id=str(resource.id), file=csv_upload("districts_v2.csv")) + + assert not tables.exists(old_table) + assert tables.names() == [ResourceDataTable.objects.get(resource=resource).table_name] + assert data_api(owner, resource.id).json()["total"] == 4 + + +@uses_data_db +def test_resetting_the_columns_switches_to_a_fresh_table( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + old_table = ResourceDataTable.objects.get(resource=resource).table_name + + run(owner, RESET, id=str(resource.id)) + + new_table = ResourceDataTable.objects.get(resource=resource).table_name + assert new_table != old_table + assert tables.names() == [new_table] + assert data_api(owner, resource.id).json()["total"] == 3 + + +@uses_data_db +def test_deleting_a_resource_removes_its_table_and_its_data( + owner: Any, dataset: Any, tables: Any +) -> None: + resource = upload(owner, dataset, "districts_v1.csv") + + run(owner, DELETE_RESOURCE, id=str(resource.id)) + + assert tables.names() == [] + assert data_api(owner, resource.id).status_code == 404 + + +@uses_data_db +def test_deleting_a_dataset_removes_every_table_it_had( + owner: Any, dataset: Any, tables: Any +) -> None: + upload(owner, dataset, "districts_v1.csv") + upload(owner, dataset, "districts_v2.csv") + assert len(tables.names()) == 2 + + run(owner, DELETE_DATASET, id=str(dataset.id)) + + assert tables.names() == [] diff --git a/tests/journeys/data_index/reupload-and-delete.py b/tests/journeys/data_index/reupload-and-delete.py new file mode 100644 index 00000000..2055b883 --- /dev/null +++ b/tests/journeys/data_index/reupload-and-delete.py @@ -0,0 +1,149 @@ +""" +Journey: upload a CSV, read its rows, replace the file while polling the data +API (it must never come back empty or half-replaced), fire two replacements at +once (the rows served must match the file the download serves), then delete +the resource and get a 404 from the data API. + +On-demand (Layer 5) — runs against a live backend. Reads from env: + KEYCLOAK_TEST_TOKEN a token for a user who can edit TEST_DATASET_ID + TEST_BASE_URL default http://localhost:8000 + TEST_DATASET_ID a draft dataset to add the resource to +Fails loudly on the first assertion that fails. + +Usage: + KEYCLOAK_TEST_TOKEN=... TEST_DATASET_ID=... TEST_BASE_URL=http://localhost:8000 \ + python tests/journeys/data_index/reupload-and-delete.py + +Afterwards, at least an hour later (the orphan list skips per-upload tables +built within the last hour, as their upload may still be switching), the +operator runs `python manage.py migrate_data_table_names` (a dry run) on the +server and checks the report lists no orphan from the racing uploads. +Docs: api/utils/data_indexing_architecture.md +""" + +import csv +import io +import json +import os +import threading + +import requests + +base = os.environ.get("TEST_BASE_URL", "http://localhost:8000") +token = os.environ["KEYCLOAK_TEST_TOKEN"] +dataset_id = os.environ["TEST_DATASET_ID"] +headers = {"Authorization": f"Bearer {token}"} +graphql = f"{base}/api/graphql" + +CREATE = """mutation ($ds: UUID!, $file: Upload!) { + createFileResources(fileResourceInput: {dataset: $ds, files: [$file]}) { id } +}""" +REPLACE = """mutation ($id: UUID!, $file: Upload!) { + updateFileResource(fileResourceInput: {id: $id, file: $file}) { __typename } +}""" +DELETE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" + + +def make_csv(label, rows): + """A CSV whose every row carries the label, so a mix of two files is visible.""" + lines = ["district,label,value"] + [f"District {i},{label},{i}" for i in range(rows)] + return ("\n".join(lines) + "\n").encode() + + +def upload(query, variables, file_bytes): + """Post a GraphQL mutation with one file, using the GraphQL multipart spec.""" + operations = {"query": query, "variables": {**variables, "file": None}} + res = requests.post( + graphql, + data={"operations": json.dumps(operations), "map": json.dumps({"0": ["variables.file"]})}, + files={"0": ("journey.csv", file_bytes, "text/csv")}, + headers=headers, + timeout=300, + ) + assert res.status_code == 200, res.text + body = res.json() + assert not body.get("errors"), body["errors"] + return body["data"] + + +def served_labels(resource_id): + """The labels the data API serves for the resource (all rows), or None on an error.""" + res = requests.get( + f"{base}/api/resources/{resource_id}/data/", + params={"limit": 10000, "columns": "label"}, + headers=headers, + timeout=60, + ) + if res.status_code != 200: + return None + return {row[0] for row in res.json()["rows"]}, res.json()["total"] + + +def downloaded_label(resource_id): + """The label in the file the download endpoint serves.""" + res = requests.get(f"{base}/api/download/resource/{resource_id}", headers=headers, timeout=60) + assert res.status_code == 200, res.text + rows = list(csv.DictReader(io.StringIO(res.content.decode()))) + labels = {row["label"] for row in rows} + assert len(labels) == 1, labels + return labels.pop() + + +# 1. Upload a CSV and read its rows back. +created = upload(CREATE, {"ds": dataset_id}, make_csv("first", 500)) +resource_id = created["createFileResources"][0]["id"] +assert served_labels(resource_id) == ({"first"}, 500) + +# 2. Replace it while polling: every read is the complete old or the complete new file. +seen = [] +done = threading.Event() + + +def poll(): + while not done.is_set(): + seen.append(served_labels(resource_id)) + + +poller = threading.Thread(target=poll) +poller.start() +upload(REPLACE, {"id": resource_id}, make_csv("second", 800)) +done.set() +poller.join() +assert seen, "the poller never read anything" +for reading in seen: + assert reading in (({"first"}, 500), ({"second"}, 800)), reading +assert served_labels(resource_id) == ({"second"}, 800) + +# 3. Two replacements at once: whichever file the download serves, the data API serves too. +race_errors = [] + + +def race(label): + try: + upload(REPLACE, {"id": resource_id}, make_csv(label, 300)) + except Exception as error: # surfaced below: a thread's exception is otherwise lost + race_errors.append(error) + + +racers = [threading.Thread(target=race, args=(label,)) for label in ("third", "fourth")] +for racer in racers: + racer.start() +for racer in racers: + racer.join() +assert not race_errors, race_errors +winner = downloaded_label(resource_id) +assert served_labels(resource_id) == ({winner}, 300), (winner, served_labels(resource_id)) + +# 4. Delete the resource: its data is gone. +res = requests.post( + graphql, json={"query": DELETE, "variables": {"id": resource_id}}, headers=headers, timeout=60 +) +assert res.status_code == 200 and not res.json().get("errors"), res.text +gone = requests.get(f"{base}/api/resources/{resource_id}/data/", headers=headers, timeout=60) +assert gone.status_code == 404, gone.text + +print("PASS: reupload-and-delete") +print( + "In an hour, run `python manage.py migrate_data_table_names` and check the report " + "lists no orphans (younger per-upload tables are skipped by design)." +) diff --git a/tests/schema/test_dataset_access_rules.py b/tests/schema/test_dataset_access_rules.py new file mode 100644 index 00000000..2ad1e302 --- /dev/null +++ b/tests/schema/test_dataset_access_rules.py @@ -0,0 +1,282 @@ +"""One dataset edit rule, any-grant view rule, organisation listings for +members only, and refused changes are logged.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Dataset, Organization, Resource +from api.schema.dataset_schema import UpdateDatasetPermission +from api.schema.schema import schema +from api.utils.dataset_access import ( + can_change_dataset, + can_view_dataset, + viewable_datasets_q, +) +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + + +def ctx(user, org=None): + return types.SimpleNamespace( + user=user, + context={"organization": org, "auth_token": "CANARY-TOKEN"} if org else {}, + request=None, + response=None, + ) + + +def info(user): + return types.SimpleNamespace(context=ctx(user), field_name="updateDataset") + + +def run(user, query, org=None, **variables): + return schema.execute_sync(query, variable_values=variables, context_value=ctx(user, org)) + + +@pytest.fixture +def w(db): + roles = { + "admin": Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ), + "editor": Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True), + "viewer": Role.objects.create(name="viewer", can_view=True), + "manager": Role.objects.create( + name="manager", can_view=True, can_change=True, can_delete=True + ), + "ds_editor": Role.objects.create(name="dataset-editor", can_view=True, can_change=True), + "blind": Role.objects.create(name="blind", can_view=False), + } + org = Organization.objects.create(name="Org", description="o", slug="org") + other = Organization.objects.create(name="Other", description="o", slug="other") + names = [ + "owner", + "admin", + "editor", + "viewer", + "manager", + "other_admin", + "stranger", + "dp_change", + "dp_elsewhere", + "viewer_blind_dp", + "solo", + ] + u = { + n: User.objects.create(username=f"u-{n}-name", email=f"{n}@example.org", keycloak_id=n) + for n in names + } + u["superuser"] = User.objects.create(username="su", keycloak_id="su", is_superuser=True) + for n, r, o in [ + ("admin", "admin", org), + ("editor", "editor", org), + ("viewer", "viewer", org), + ("manager", "manager", org), + ("viewer_blind_dp", "viewer", org), + ("other_admin", "admin", other), + ]: + OrganizationMembership.objects.create(user=u[n], organization=o, role=roles[r]) + + def ds(slug, status="DRAFT", **kw): + return Dataset.objects.create(title=slug, slug=slug, description="d", status=status, **kw) + + d = types.SimpleNamespace( + draft=ds("draft", user=u["owner"], organization=org), + published=ds("pub", "PUBLISHED", user=u["owner"], organization=org), + archived=ds("arch", "ARCHIVED", user=u["owner"], organization=org), + solo=ds("solo", user=u["solo"]), + ) + for x in (d.draft, d.published, d.archived): + DatasetPermission.objects.create(user=u["dp_change"], dataset=x, role=roles["ds_editor"]) + DatasetPermission.objects.create( + user=u["dp_elsewhere"], dataset=d.solo, role=roles["ds_editor"] + ) + DatasetPermission.objects.create( + user=u["viewer_blind_dp"], dataset=d.draft, role=roles["blind"] + ) + return types.SimpleNamespace(u=u, d=d, org=org, other=other) + + +# ---- the edit rule --------------------------------------------------------- + +EDIT = { + # user: (draft, published, archived) + "owner": (True, True, True), + "admin": (True, True, True), + "editor": (True, True, True), + "viewer": (False, False, False), + "manager": (False, False, False), # org edit rights follow the role name + "other_admin": (False, False, False), + "stranger": (False, False, False), + "dp_change": (True, False, False), + "dp_elsewhere": (False, False, False), + "superuser": (True, True, True), +} + + +@pytest.mark.parametrize("who", EDIT) +def test_edit_rule(w, who): + got = tuple(can_change_dataset(w.u[who], x) for x in (w.d.draft, w.d.published, w.d.archived)) + assert got == EDIT[who] + + +@pytest.mark.parametrize("who", EDIT) +def test_update_dataset_permission_uses_edit_rule(w, who): + perm = UpdateDatasetPermission() + for x, expected in zip((w.d.draft, w.d.published, w.d.archived), EDIT[who]): + for key in ("update_dataset_input", "update_metadata_input", "update_input"): + got = perm.has_permission( + None, info(w.u[who]), **{key: types.SimpleNamespace(dataset=x.id)} + ) + assert bool(got) == expected, (who, x.status, key) + + +def test_update_dataset_permission_edges(w): + perm = UpdateDatasetPermission() + editor, su = w.u["editor"], w.u["superuser"] + unknown = types.SimpleNamespace(dataset=uuid.uuid4()) + assert not perm.has_permission(None, info(AnonymousUser()), update_input=unknown) + assert not perm.has_permission(None, info(editor)) + assert not perm.has_permission(None, info(editor), update_input=types.SimpleNamespace()) + assert not perm.has_permission(None, info(editor), update_input=unknown) + # Superusers still pass before the lookup, as before. + assert perm.has_permission(None, info(su)) + assert perm.has_permission(None, info(su), update_input=unknown) + + +def test_null_owner_and_org_dataset(w): + orphan = Dataset.objects.create(title="o", slug="orphan", description="d") + DatasetPermission.objects.create( + user=w.u["dp_change"], dataset=orphan, role=Role.objects.get(name="dataset-editor") + ) + assert not can_change_dataset(w.u["editor"], orphan) + assert not can_view_dataset(w.u["stranger"], orphan) + assert can_change_dataset(w.u["dp_change"], orphan) + assert can_change_dataset(w.u["superuser"], orphan) + + +# ---- the view rule --------------------------------------------------------- + +VIEW_DRAFT = { + "owner": True, + "admin": True, + "editor": True, + "viewer": True, + "manager": True, + "other_admin": False, + "stranger": False, + "dp_change": True, + "dp_elsewhere": False, + "viewer_blind_dp": True, # any one grant is enough + "superuser": True, +} + + +@pytest.mark.parametrize("who", VIEW_DRAFT) +def test_view_rule_matches_list_filter(w, who): + user = w.u[who] + for x in (w.d.draft, w.d.archived): + expected = VIEW_DRAFT[who] + assert can_view_dataset(user, x) == expected, (who, x.status) + listed = Dataset.objects.filter(viewable_datasets_q(user), id=x.id).exists() + assert listed == expected, (who, x.status) + assert can_view_dataset(user, w.d.published) + + +def test_anonymous_view_no_queries(w, django_assert_num_queries): + with django_assert_num_queries(0): + assert can_view_dataset(AnonymousUser(), w.d.published) + assert not can_view_dataset(AnonymousUser(), w.d.draft) + assert not can_change_dataset(AnonymousUser(), w.d.draft) + + +# ---- the organisation header ---------------------------------------------- + +DATASETS = "query ($p: Boolean) { datasets(includePublic: $p) { id } }" +TABLE = "query { datasetsTable(limit: 50) { data { id } totalItemsCount } }" +NESTED = "query { datasets { resources { id name } } }" + + +def ids(r, field="datasets"): + assert r.errors is None, r.errors + rows = r.data[field]["data"] if field == "datasetsTable" else r.data[field] + return {row["id"] for row in rows} + + +@pytest.mark.parametrize("who", ["anon", "stranger", "other_admin"]) +def test_header_does_not_list_another_orgs_drafts(w, who): + user = AnonymousUser() if who == "anon" else w.u[who] + Resource.objects.create(name="secret.csv", dataset=w.d.draft) + assert ids(run(user, DATASETS, org=w.org)) == set() + assert ids(run(user, TABLE, org=w.org), "datasetsTable") == set() + r = run(user, NESTED, org=w.org) + assert "secret.csv" not in str(r.data) and r.errors is None + # includePublic still adds published datasets, and only those. + assert ids(run(user, DATASETS, org=w.org, p=True)) == {str(w.d.published.id)} + + +@pytest.mark.parametrize("who", ["viewer", "editor", "admin", "superuser"]) +def test_members_list_their_orgs_datasets(w, who): + expected = {str(x.id) for x in (w.d.draft, w.d.published, w.d.archived)} + assert ids(run(w.u[who], DATASETS, org=w.org)) == expected + assert ids(run(w.u[who], TABLE, org=w.org), "datasetsTable") == expected + + +def test_no_header_lists_own_solo_datasets(w): + assert ids(run(w.u["solo"], DATASETS)) == {str(w.d.solo.id)} + + +# ---- refusals are logged --------------------------------------------------- + +RENAME = """mutation ($id: UUID!) { + updateFileResource(fileResourceInput: {id: $id, name: "x"}) { ... on TypeResource { name } } +}""" + + +def denial_records(caplog): + out = [] + for rec in caplog.records: + msg = rec.msg if isinstance(rec.msg, dict) else {} + if msg.get("event") == "dataset access denied": + out.append(msg) + return out + + +@pytest.mark.parametrize( + "who,reason", + [("anon", "not_signed_in"), ("other_admin", "not_allowed"), ("unknown", "target_not_found")], +) +def test_refusal_logged_once_without_personal_data(w, caplog, who, reason): + res = Resource.objects.create(name="f.csv", dataset=w.d.draft) + user = AnonymousUser() if who == "anon" else w.u["other_admin"] + rid = uuid.uuid4() if who == "unknown" else res.id + caplog.set_level("WARNING") + r = run(user, RENAME, org=w.org, id=str(rid)) + assert r.errors + res.refresh_from_db() + assert res.name == "f.csv" + recs = denial_records(caplog) + assert len(recs) == 1 + assert recs[0]["reason"] == reason + assert recs[0]["operation"] == "updateFileResource" + assert recs[0]["user_id"] == getattr(user, "id", None) + for rec in caplog.records: + text = rec.getMessage() + str(rec.msg) + for secret in ("CANARY-TOKEN", "other_admin@example.org", "u-other_admin-name"): + assert secret not in text + + +def test_allowed_change_logs_nothing(w, caplog): + res = Resource.objects.create(name="f.csv", dataset=w.d.draft) + caplog.set_level("WARNING") + r = run(w.u["editor"], RENAME, org=w.org, id=str(res.id)) + assert r.errors is None, r.errors + assert denial_records(caplog) == [] + + +def test_old_resource_permission_class_removed(): + import authorization.permissions as p + + assert not hasattr(p, "ResourcePermissionGraphQL") diff --git a/tests/schema/test_dataset_part_permissions.py b/tests/schema/test_dataset_part_permissions.py new file mode 100644 index 00000000..842ede7f --- /dev/null +++ b/tests/schema/test_dataset_part_permissions.py @@ -0,0 +1,372 @@ +"""Charts, chart images, access models and prompt file +details need edit rights on their dataset; drafts' ones are hidden from people +who cannot see the draft.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.test import RequestFactory + +from api.models import ( + AccessModel, + Dataset, + Organization, + PromptResource, + Resource, + ResourceChartDetails, + ResourceChartImage, + ResourceSchema, +) +from api.schema.schema import schema +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + + +def run(user, query, **variables): + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def world(db): + org = Organization.objects.create(name="Org", description="o", slug="org") + other_org = Organization.objects.create(name="Other", description="o", slug="other-org") + editor_role = Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True) + viewer_role = Role.objects.create(name="viewer", can_view=True) + users = { + name: User.objects.create(username=name, keycloak_id=name) + for name in ["owner", "editor", "viewer", "outsider", "shared"] + } + users["admin"] = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + OrganizationMembership.objects.create(user=users["editor"], organization=org, role=editor_role) + OrganizationMembership.objects.create(user=users["viewer"], organization=org, role=viewer_role) + OrganizationMembership.objects.create( + user=users["outsider"], organization=other_org, role=editor_role + ) + draft = Dataset.objects.create( + title="Draft", slug="draft", description="d", user=users["owner"], organization=org + ) + published = Dataset.objects.create( + title="Pub", + slug="pub", + description="d", + user=users["owner"], + organization=org, + status="PUBLISHED", + ) + # The outsider's own dataset, for "move it into my dataset" attempts. + theirs = Dataset.objects.create( + title="Theirs", + slug="theirs", + description="d", + user=users["outsider"], + organization=other_org, + ) + DatasetPermission.objects.create(user=users["shared"], dataset=draft, role=editor_role) + draft_res = Resource.objects.create(name="d.csv", dataset=draft) + pub_res = Resource.objects.create(name="p.csv", dataset=published) + their_res = Resource.objects.create(name="t.csv", dataset=theirs) + return types.SimpleNamespace( + users=users, + draft=draft, + published=published, + theirs=theirs, + draft_res=draft_res, + pub_res=pub_res, + their_res=their_res, + draft_chart=ResourceChartDetails.objects.create(name="dc", resource=draft_res), + pub_chart=ResourceChartDetails.objects.create(name="pc", resource=pub_res), + draft_image=ResourceChartImage.objects.create(name="di", dataset=draft), + pub_image=ResourceChartImage.objects.create(name="pi", dataset=published), + draft_am=AccessModel.objects.create(name="da", dataset=draft), + pub_am=AccessModel.objects.create(name="pa", dataset=published), + ) + + +def user_for(world, who): + return AnonymousUser() if who == "anonymous" else world.users[who] + + +def denied(result): + """A permission refusal, as a top-level error or a handled OperationInfo.""" + text = str(result.errors) if result.errors else str(result.data) + return "permission" in text + + +def u(ids): + return {k: str(v) for k, v in ids.items()} + + +CHART = "... on TypeResourceChart { id } ... on OperationInfo { messages { message } }" +IMAGE = "... on TypeResourceChartImage { id } ... on OperationInfo { messages { message } }" +AM = "... on TypeAccessModel { id } ... on OperationInfo { messages { message } }" + +# (query, variables from world, check the object is untouched afterwards) +MUTATIONS = { + "addResourceChart": ( + f"mutation ($r: UUID!) {{ addResourceChart(resource: $r) {{ {CHART} }} }}", + lambda w: {"r": w.pub_res.id}, + ), + "createResourceChart": ( + f"mutation ($r: UUID!) {{ createResourceChart(chartInput: {{resource: $r}}) {{ {CHART} }} }}", + lambda w: {"r": w.pub_res.id}, + ), + "editResourceChart": ( + f"""mutation ($c: UUID!, $r: UUID!) {{ + editResourceChart(chartInput: {{chartId: $c, resource: $r, name: "x"}}) {{ {CHART} }} + }}""", + lambda w: {"c": w.pub_chart.id, "r": w.pub_res.id}, + ), + "deleteResourceChart": ( + "mutation ($c: UUID!) { deleteResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "publishResourceChart": ( + "mutation ($c: UUID!) { publishResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "unpublishResourceChart": ( + "mutation ($c: UUID!) { unpublishResourceChart(chartId: $c) }", + lambda w: {"c": w.pub_chart.id}, + ), + "createResourceChartImage": ( + f"mutation ($d: UUID!) {{ createResourceChartImage(input: {{dataset: $d}}) {{ {IMAGE} }} }}", + lambda w: {"d": w.published.id}, + ), + "addResourceChartImage": ( + f"mutation ($d: UUID!) {{ addResourceChartImage(dataset: $d) {{ {IMAGE} }} }}", + lambda w: {"d": w.published.id}, + ), + "updateResourceChartImage": ( + f"""mutation ($i: UUID!, $d: UUID!) {{ + updateResourceChartImage(input: {{id: $i, dataset: $d, name: "x"}}) {{ {IMAGE} }} + }}""", + lambda w: {"i": w.pub_image.id, "d": w.published.id}, + ), + "publishResourceChartImage": ( + "mutation ($i: UUID!) { publishResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "unpublishResourceChartImage": ( + "mutation ($i: UUID!) { unpublishResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "deleteResourceChartImage": ( + "mutation ($i: UUID!) { deleteResourceChartImage(resourceChartImageId: $i) }", + lambda w: {"i": w.pub_image.id}, + ), + "createAccessModel": ( + f"""mutation ($d: UUID!) {{ + createAccessModel(accessModelInput: {{dataset: $d, name: "x", description: null, type: PUBLIC, resources: []}}) + {{ {AM} }} + }}""", + lambda w: {"d": w.published.id}, + ), + "editAccessModel": ( + f"""mutation ($a: UUID!, $d: UUID!) {{ + editAccessModel(accessModelInput: {{ + accessModelId: $a, dataset: $d, name: "x", description: null, type: null, resources: null + }}) {{ {AM} }} + }}""", + lambda w: {"a": w.pub_am.id, "d": w.published.id}, + ), + "deleteAccessModel": ( + "mutation ($a: UUID!) { deleteAccessModel(accessModelId: $a) }", + lambda w: {"a": w.pub_am.id}, + ), + "updatePromptResource": ( + """mutation ($r: UUID!) { + updatePromptResource(updateInput: {resource: $r, promptCount: 5}) { success } + }""", + lambda w: {"r": w.draft_res.id}, + ), +} + + +def untouched(w): + w.pub_chart.refresh_from_db() + w.pub_image.refresh_from_db() + w.pub_am.refresh_from_db() + return ( + w.pub_chart.name == "pc" + and w.pub_chart.status == "DRAFT" + and w.pub_image.name == "pi" + and w.pub_am.name == "pa" + and ResourceChartDetails.objects.count() == 2 + and ResourceChartImage.objects.count() == 2 + and AccessModel.objects.count() == 2 + and not PromptResource.objects.exists() + ) + + +@pytest.mark.parametrize("who", ["anonymous", "outsider", "viewer"]) +@pytest.mark.parametrize("name", list(MUTATIONS)) +def test_people_without_edit_rights_are_refused(world, who, name): + query, variables = MUTATIONS[name] + r = run(user_for(world, who), query, **u(variables(world))) + assert denied(r), (who, name, r.errors, r.data) + assert untouched(world) + + +@pytest.mark.parametrize("who", ["owner", "editor", "admin"]) +@pytest.mark.parametrize("name", list(MUTATIONS)) +def test_people_with_edit_rights_are_let_through(world, who, name): + query, variables = MUTATIONS[name] + r = run(world.users[who], query, **u(variables(world))) + assert not denied(r), (who, name, r.errors, r.data) + + +@pytest.mark.parametrize( + "query", + [ + "mutation ($i: UUID!) { deleteResourceChart(chartId: $i) }", + "mutation ($i: UUID!) { deleteResourceChartImage(resourceChartImageId: $i) }", + "mutation ($i: UUID!) { deleteAccessModel(accessModelId: $i) }", + ], +) +def test_missing_id_is_refused_like_a_forbidden_one(world, query): + r = run(AnonymousUser(), query, i=str(uuid.uuid4())) + assert denied(r) + assert "does not exist" not in str(r.errors) + + +# Moving something out of a dataset you cannot edit, into one you can. + + +def test_cannot_move_someone_elses_chart_onto_my_resource(world): + query, _ = MUTATIONS["editResourceChart"] + r = run(world.users["outsider"], query, c=str(world.pub_chart.id), r=str(world.their_res.id)) + assert denied(r) + world.pub_chart.refresh_from_db() + assert world.pub_chart.resource_id == world.pub_res.id + + +def test_cannot_move_someone_elses_chart_image_into_my_dataset(world): + query, _ = MUTATIONS["updateResourceChartImage"] + r = run(world.users["outsider"], query, i=str(world.pub_image.id), d=str(world.theirs.id)) + assert denied(r) + world.pub_image.refresh_from_db() + assert world.pub_image.dataset_id == world.published.id + + +def test_cannot_take_someone_elses_access_model_by_naming_my_dataset(world): + query, _ = MUTATIONS["editAccessModel"] + r = run(world.users["outsider"], query, a=str(world.pub_am.id), d=str(world.theirs.id)) + assert denied(r) + + +def test_access_model_cannot_include_another_datasets_resource(world): + r = run( + world.users["outsider"], + f"""mutation ($d: UUID!, $r: UUID!) {{ + createAccessModel(accessModelInput: {{ + dataset: $d, name: "x", description: null, type: PUBLIC, + resources: [{{resource: $r, fields: []}}] + }}) {{ {AM} }} + }}""", + d=str(world.theirs.id), + r=str(world.draft_res.id), + ) + assert "does not exist" in str(r.errors or r.data) + assert not AccessModel.objects.filter(accessmodelresource__resource=world.draft_res).exists() + + +def test_chart_cannot_use_another_resources_column(world): + column = ResourceSchema.objects.create( + resource=world.draft_res, field_name="secret", format="STRING" + ) + r = run( + world.users["outsider"], + f"""mutation ($r: UUID!, $col: String!) {{ + createResourceChart(chartInput: {{resource: $r, options: {{xAxisColumn: $col}}}}) + {{ {CHART} }} + }}""", + r=str(world.their_res.id), + col=str(column.id), + ) + assert "TypeResourceChart" not in str(r.data) or "does not exist" in str(r.data) + assert not any( + "secret" in str(c.options) + for c in ResourceChartDetails.objects.filter(resource=world.their_res) + ) + + +# Queries. + +LISTS = { + "chartsDetails": ("query ($d: UUID!) { chartsDetails(datasetId: $d) { id } }", "draft_chart"), + "datasetResourceCharts": ( + "query ($d: UUID!) { datasetResourceCharts(datasetId: $d) { id } }", + "draft_image", + ), + "accessModelResources": ( + "query ($d: UUID!) { accessModelResources(datasetId: $d) { id } }", + "draft_am", + ), +} +ONES = { + "resourceChart": ("query ($i: UUID!) { resourceChart(chartDetailsId: $i) { id } }", "chart"), + "resourceChartImage": ("query ($i: UUID!) { resourceChartImage(imageId: $i) { id } }", "image"), + "accessModel": ("query ($i: UUID!) { accessModel(accessModelId: $i) { id } }", "am"), +} + + +@pytest.mark.parametrize("name", list(LISTS)) +def test_published_lists_stay_public(world, name): + query, _ = LISTS[name] + r = run(AnonymousUser(), query, d=str(world.published.id)) + assert r.errors is None and len(r.data[name]) == 1 + + +@pytest.mark.parametrize("name", list(ONES)) +def test_published_items_stay_public(world, name): + query, attr = ONES[name] + r = run(AnonymousUser(), query, i=str(getattr(world, f"pub_{attr}").id)) + assert r.errors is None and r.data[name]["id"] + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +@pytest.mark.parametrize("name", list(LISTS)) +def test_draft_lists_are_hidden(world, who, name): + query, _ = LISTS[name] + r = run(user_for(world, who), query, d=str(world.draft.id)) + assert r.errors is None and r.data[name] == [] + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +@pytest.mark.parametrize("name", list(ONES)) +def test_draft_items_are_hidden(world, who, name): + query, attr = ONES[name] + r = run(user_for(world, who), query, i=str(getattr(world, f"draft_{attr}").id)) + assert r.errors and "does not exist" in str(r.errors[0].message) + + +@pytest.mark.parametrize("who", ["owner", "editor", "viewer", "shared", "admin"]) +def test_drafts_visible_to_people_who_can_see_the_draft(world, who): + user = world.users[who] + for name, (query, _) in LISTS.items(): + r = run(user, query, d=str(world.draft.id)) + assert r.errors is None and len(r.data[name]) == 1, (who, name, r.errors) + for name, (query, attr) in ONES.items(): + r = run(user, query, i=str(getattr(world, f"draft_{attr}").id)) + assert r.errors is None, (who, name, r.errors) + + +def test_all_chart_images_list_hides_drafts(world): + query = "{ resourceChartImages { name } }" + r = run(AnonymousUser(), query) + assert r.errors is None and r.data["resourceChartImages"] == [{"name": "pi"}] + r = run(world.users["owner"], query) + assert sorted(i["name"] for i in r.data["resourceChartImages"]) == ["di", "pi"] + + +def test_all_chart_images_list_still_filters(world): + r = run( + world.users["owner"], + 'query { resourceChartImages(filters: {name: "di"}) { name } }', + ) + assert r.errors is None and r.data["resourceChartImages"] == [{"name": "di"}] diff --git a/tests/schema/test_dataset_public_links.py b/tests/schema/test_dataset_public_links.py new file mode 100644 index 00000000..a9a59bca --- /dev/null +++ b/tests/schema/test_dataset_public_links.py @@ -0,0 +1,98 @@ +import types + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Dataset +from api.schema.schema import schema +from api.services.metadata_export.adapter import dataset_to_record +from api.utils.enums import DatasetStatus +from authorization.models import User + +QUERY = """ +query($id: UUID, $pid: String) { + getDataset(datasetId: $id, publicId: $pid) { id title publicId urlSlug } +} +""" + + +def run(query, user, variables): + ctx = types.SimpleNamespace(user=user, context={}) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def published(db): + return Dataset.objects.create(title="City Air Quality Index", status=DatasetStatus.PUBLISHED) + + +@pytest.fixture +def draft(db): + return Dataset.objects.create(title="Secret draft", status=DatasetStatus.DRAFT) + + +def test_new_dataset_gets_public_id_and_link(published) -> None: + assert len(published.public_id) == 12 + assert published.url_slug == f"city-air-quality-index-{published.public_id}" + + +def test_rename_keeps_public_id(published) -> None: + old = published.public_id + published.title = "Air Quality Index for Cities" + published.save() + published.refresh_from_db() + assert published.public_id == old + assert published.url_slug == f"air-quality-index-for-cities-{old}" + + +def test_get_by_public_id_anonymous(published) -> None: + for value in [published.public_id, published.url_slug, f"old-words-{published.public_id}"]: + res = run(QUERY, AnonymousUser(), {"pid": value}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["id"] == str(published.id) + assert res.data["getDataset"]["urlSlug"] == published.url_slug + + +def test_get_by_uuid_still_works(published) -> None: + res = run(QUERY, AnonymousUser(), {"id": str(published.id)}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["publicId"] == published.public_id + + +def test_unknown_or_bad_public_id_gives_null(published) -> None: + user = User.objects.create(username="u1", keycloak_id="u1") + for value in ["aaaaaaaaaaaa", "not-a-link", str(published.id)]: + res = run(QUERY, user, {"pid": value}) + assert res.errors is None, res.errors + assert res.data["getDataset"] is None + + +def test_draft_by_public_id_needs_login(draft) -> None: + res = run(QUERY, AnonymousUser(), {"pid": draft.public_id}) + assert res.errors + assert res.data is None or res.data["getDataset"] is None + + +def test_draft_by_public_id_other_user_denied(draft) -> None: + stranger = User.objects.create(username="u2", keycloak_id="u2") + res = run(QUERY, stranger, {"pid": draft.public_id}) + assert res.errors + + +def test_draft_by_public_id_superuser(draft) -> None: + admin = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + res = run(QUERY, admin, {"pid": draft.public_id}) + assert res.errors is None, res.errors + assert res.data["getDataset"]["publicId"] == draft.public_id + + +def test_export_landing_link_uses_url_slug(published) -> None: + record = dataset_to_record(published) + assert record["landing_page"].endswith(f"/datasets/{published.url_slug}") + + +def test_hindi_title_link_is_just_the_id(db) -> None: + ds = Dataset.objects.create(title="जनगणना डेटा", status=DatasetStatus.PUBLISHED) + assert ds.url_slug == ds.public_id + res = run(QUERY, AnonymousUser(), {"pid": ds.url_slug}) + assert res.data["getDataset"]["id"] == str(ds.id) diff --git a/tests/schema/test_event_cover.py b/tests/schema/test_event_cover.py new file mode 100644 index 00000000..138e4d0f --- /dev/null +++ b/tests/schema/test_event_cover.py @@ -0,0 +1,175 @@ +"""Luma cover image on the event search document.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from api.models import Event +from api.utils.enums import EventStatus +from api.utils.luma_cover import cover_from_response, fetch_luma_cover, luma_event_id +from search.documents import EventDocument + +EMBED = "https://lu.ma/embed/event/evt-abc123/simple" +COVER = "https://images.lumacdn.com/gallery-images/rc/5f979993.png" +NEW_COVER = "https://images.lumacdn.com/gallery-images/7t/2758125c.png" + + +def body(event: object) -> dict: + return {"api_id": "x", "event": event} + + +def response(status: int = 200, payload: object = None, raw: bytes = b"") -> MagicMock: + resp = MagicMock() + resp.status_code = status + resp.iter_content.return_value = [raw or json.dumps(payload).encode()] + resp.__enter__.return_value = resp + return resp + + +# ── embed id ───────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "src, expected", + [ + (EMBED, "evt-abc123"), + ("https://luma.com/embed/event/evt-abc123", "evt-abc123"), + ("https://luma.com/embed/calendar/cal-x/events", None), + ("https://luma.com/embed/event/../../admin/simple", None), + ("", None), + (None, None), + (123, None), + ], +) +def test_event_id_comes_only_from_an_event_embed(src, expected): + assert luma_event_id(src) == expected + + +# ── response parsing ───────────────────────────────────────────────────── + + +def test_cover_is_read_from_the_event(): + assert ( + cover_from_response(body({"api_id": "evt-abc123", "cover_url": COVER}), "evt-abc123") + == COVER + ) + + +def test_cover_of_another_event_is_ignored(): + assert ( + cover_from_response(body({"api_id": "evt-zzz", "cover_url": COVER}), "evt-abc123") is None + ) + + +@pytest.mark.parametrize( + "url", + [ + "http://images.lumacdn.com/x.png", + "https://evil.example.com/x.png", + "https://images.lumacdn.com.evil.example/x.png", + "https://user@images.lumacdn.com/x.png", + "", + None, + 123, + ], +) +def test_only_https_luma_cdn_images_are_accepted(url): + assert ( + cover_from_response(body({"api_id": "evt-abc123", "cover_url": url}), "evt-abc123") is None + ) + + +@pytest.mark.parametrize("data", [{}, [], "x", None, body(None), body("x")]) +def test_unexpected_body_gives_no_cover(data): + assert cover_from_response(data, "evt-abc123") is None + + +# ── fetching ───────────────────────────────────────────────────────────── + + +@patch("api.utils.luma_cover.requests.get") +def test_fetch_calls_event_get_with_the_id_without_redirects(get): + get.return_value = response(payload=body({"api_id": "evt-abc123", "cover_url": COVER})) + assert fetch_luma_cover(EMBED) == COVER + assert get.call_args.args[0] == "https://api2.luma.com/event/get" + assert get.call_args.kwargs["params"] == {"event_api_id": "evt-abc123"} + assert get.call_args.kwargs["allow_redirects"] is False + assert get.call_args.kwargs["timeout"] <= 5 + + +@pytest.mark.parametrize( + "resp", + [ + requests.Timeout(), + requests.ConnectionError(), + response(status=404, payload={"message": "not found"}), + response(status=302, payload={}), + response(status=500, payload={}), + response(raw=b"not json"), + ], +) +@patch("api.utils.luma_cover.requests.get") +def test_fetch_failures_give_null(get, resp): + if isinstance(resp, Exception): + get.side_effect = resp + else: + get.return_value = resp + assert fetch_luma_cover(EMBED) is None + + +@patch("api.utils.luma_cover._MAX_BYTES", 10) +@patch("api.utils.luma_cover.requests.get") +def test_oversized_response_is_not_read(get): + get.return_value = response(payload=body({"api_id": "evt-abc123", "cover_url": COVER})) + assert fetch_luma_cover(EMBED) is None + + +@patch("api.utils.luma_cover.requests.get") +def test_no_request_without_an_event_embed(get): + assert fetch_luma_cover(None) is None + assert fetch_luma_cover("https://luma.com/embed/calendar/cal-x/events") is None + get.assert_not_called() + + +# ── search document ────────────────────────────────────────────────────── + + +def event(**kwargs) -> Event: + defaults = dict(name="Meetup", status=EventStatus.PUBLISHED, luma_embed={"src": EMBED}) + defaults.update(kwargs) + return Event(**defaults) + + +@patch("search.documents.event_document.fetch_luma_cover", return_value=COVER) +def test_document_asks_luma_for_the_stored_embed(fetch): + assert EventDocument().prepare(event())["cover_url"] == COVER + fetch.assert_called_once_with(EMBED) + + +@patch("search.documents.event_document.fetch_luma_cover") +def test_document_asks_luma_on_every_build(fetch): + """No value is carried over: each build (publish, edit, deploy populate) asks again.""" + doc, instance = EventDocument(), event() + fetch.return_value = COVER + assert doc.prepare(instance)["cover_url"] == COVER + fetch.return_value = NEW_COVER + assert doc.prepare(instance)["cover_url"] == NEW_COVER + fetch.return_value = None + assert doc.prepare(instance)["cover_url"] is None + assert fetch.call_count == 3 + + +@pytest.mark.parametrize("embed", [{}, None, "not a dict", {"src": None}]) +def test_document_without_embed_has_null_cover(embed): + with patch("api.utils.luma_cover.requests.get") as get: + assert EventDocument().prepare(event(luma_embed=embed))["cover_url"] is None + get.assert_not_called() + + +@patch("search.documents.event_document.fetch_luma_cover", return_value=COVER) +def test_document_does_not_carry_the_embed(fetch): + prepared = EventDocument().prepare(event()) + assert "luma_embed" not in prepared + assert EMBED not in json.dumps(prepared, default=str) diff --git a/tests/schema/test_event_schema.py b/tests/schema/test_event_schema.py new file mode 100644 index 00000000..8590da57 --- /dev/null +++ b/tests/schema/test_event_schema.py @@ -0,0 +1,400 @@ +"""GraphQL tests for the event provider.""" + +import types +from datetime import date +from unittest.mock import patch + +import pytest +from django.contrib.auth.models import AnonymousUser + +from api.models import Event +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.enums import EventStatus +from api.utils.luma import extract_luma_embed_url, parse_luma_embed +from authorization.models import OrganizationMembership, Role, User + +EMBED = "https://lu.ma/embed/event/evt-abc/simple" +IFRAME = ( + f'' +) + + +@pytest.fixture(autouse=True) +def _no_activity_recording(): + with patch("api.schema.base_mutation.record_activity", return_value=None): + yield + + +@pytest.fixture(autouse=True) +def _no_luma_requests(): + """Publishing builds the search document, which asks Luma for the cover.""" + with patch("search.documents.event_document.fetch_luma_cover", return_value=None): + yield + + +@pytest.fixture +def roles(db): + admin = Role.objects.create( + name="admin", can_view=True, can_add=True, can_change=True, can_delete=True + ) + auditor = Role.objects.create( + name="auditor", can_view=True, can_add=False, can_change=False, can_delete=False + ) + return {"admin": admin, "auditor": auditor} + + +@pytest.fixture +def org_a(db): + return Organization.objects.create(name="Org A", description="a", slug="org-a") + + +@pytest.fixture +def org_b(db): + return Organization.objects.create(name="Org B", description="b", slug="org-b") + + +def _member(user, org, role): + OrganizationMembership.objects.create(user=user, organization=org, role=role) + return user + + +@pytest.fixture +def org_a_admin(roles, org_a): + return _member( + User.objects.create(username="event_a_admin", keycloak_id="event_a_admin"), + org_a, + roles["admin"], + ) + + +@pytest.fixture +def org_a_auditor(roles, org_a): + return _member( + User.objects.create(username="event_a_auditor", keycloak_id="event_a_auditor"), + org_a, + roles["auditor"], + ) + + +@pytest.fixture +def org_b_admin(roles, org_b): + return _member( + User.objects.create(username="event_b_admin", keycloak_id="event_b_admin"), + org_b, + roles["admin"], + ) + + +@pytest.fixture +def individual(db): + return User.objects.create(username="event_solo", keycloak_id="event_solo") + + +def ctx(user, organization=None): + return types.SimpleNamespace( + user=user, + context={"organization": organization} if organization else {}, + ) + + +def run(query, context, variables=None): + return schema.execute_sync(query, variable_values=variables or {}, context_value=context) + + +def _make_event(user, org, **overrides): + values = { + "name": "Existing meetup", + "location": "Pune", + "start_date": date(2026, 6, 1), + "end_date": date(2026, 6, 2), + "luma_embed": {"src": EMBED}, + "organization": org, + "user": user, + "status": EventStatus.DRAFT, + } + values.update(overrides) + return Event.objects.create(**values) + + +CREATE = """ +mutation Create($input: CreateEventInput!) { + createEvent(input: $input) { + success + errors { nonFieldErrors fieldErrors { field messages } } + data { id status isIndividualEvent organization { id } user { id } } + } +} +""" + +UPDATE = """ +mutation Update($input: UpdateEventInput!) { + updateEvent(input: $input) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id name location startDate endDate lumaEmbed { src width height style allowfullscreen } } + } +} +""" + +PUBLISH = """ +mutation Publish($id: UUID!) { + publishEvent(eventId: $id) { + success + errors { fieldErrors { field messages } nonFieldErrors } + data { id status } + } +} +""" + +TABLE = """ +query Events($filters: [FilterSpec!]) { + eventsTable(filters: $filters) { + totalItemsCount + data { id status name } + } +} +""" + +GET = """ +query Get($id: UUID!) { + event(eventId: $id) { id status } +} +""" + +PUBLISHED = """ +query Published($organizationSlug: String) { + publishedEvents(organizationSlug: $organizationSlug) { + totalItemsCount + data { name urlSlug status organization { slug } } + } +} +""" + +BY_SLUG = """ +query BySlug($publicId: String!) { + eventBySlug(publicId: $publicId) { name urlSlug status lumaEmbed { src } } +} +""" + + +def test_iframe_keeps_presentation_attributes(): + embed = parse_luma_embed(IFRAME) + assert embed is not None + assert embed["src"] == EMBED + assert embed["width"] == "600" + assert embed["height"] == "450" + assert embed["allowfullscreen"] is True + assert "border-radius" in str(embed["style"]) + + +def test_iframe_src_becomes_a_canonical_embed_url(): + assert extract_luma_embed_url(IFRAME) == EMBED + + +def test_script_and_event_handlers_are_rejected(): + assert parse_luma_embed(f'') is None + assert parse_luma_embed(f'') is None + + +def test_non_embed_luma_page_is_rejected(): + assert extract_luma_embed_url("https://lu.ma/community-meetup") is None + assert extract_luma_embed_url("http://lu.ma/embed/event/evt-abc/simple") is None + assert extract_luma_embed_url("https://evil.example/embed/event/evt-abc") is None + + +@pytest.mark.django_db +class TestEventSchema: + def test_org_member_creates_an_org_draft(self, org_a_admin, org_a): + result = run(CREATE, ctx(org_a_admin, org_a), {"input": {}}) + + assert result.errors is None + payload = result.data["createEvent"] + assert payload["success"] is True + assert payload["data"]["status"] == "DRAFT" + assert payload["data"]["organization"]["id"] == str(org_a.id) + assert payload["data"]["isIndividualEvent"] is False + + def test_individual_creates_a_user_owned_draft(self, individual): + result = run(CREATE, ctx(individual), {"input": {"name": "Solo night"}}) + + payload = result.data["createEvent"] + assert payload["success"] is True + assert payload["data"]["isIndividualEvent"] is True + assert payload["data"]["user"]["id"] == str(individual.id) + + def test_auditor_cannot_create(self, org_a_auditor, org_a): + result = run(CREATE, ctx(org_a_auditor, org_a), {"input": {}}) + + assert result.data["createEvent"]["success"] is False + assert Event.objects.count() == 0 + + def test_anonymous_cannot_create(self): + result = run(CREATE, ctx(AnonymousUser()), {"input": {}}) + + assert result.data["createEvent"]["success"] is False + + def test_invalid_embed_is_rejected(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + {"input": {"id": event_id, "lumaEmbed": "https://youtu.be/dQw4w9WgXcQ"}}, + ) + + assert result.data["updateEvent"]["success"] is False + event = Event.objects.get(id=event_id) + assert event.luma_embed == {} + + def test_iframe_is_stored_as_the_embed_url(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + { + "input": { + "id": event_id, + "name": "Community meetup", + "location": "Pune", + "startDate": "2026-06-01", + "endDate": "2026-06-02", + "lumaEmbed": IFRAME, + } + }, + ) + + payload = result.data["updateEvent"] + assert result.errors is None + assert payload["success"] is True + assert payload["data"]["lumaEmbed"]["src"] == EMBED + assert payload["data"]["lumaEmbed"]["width"] == "600" + assert payload["data"]["lumaEmbed"]["height"] == "450" + assert payload["data"]["lumaEmbed"]["allowfullscreen"] is True + + def test_end_date_before_start_is_rejected(self, individual): + created = run(CREATE, ctx(individual), {"input": {}}) + event_id = created.data["createEvent"]["data"]["id"] + + result = run( + UPDATE, + ctx(individual), + {"input": {"id": event_id, "startDate": "2026-06-02", "endDate": "2026-06-01"}}, + ) + + assert result.data["updateEvent"]["success"] is False + + def test_publish_rejects_an_incomplete_draft(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, location="", luma_embed={}) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(event.id)}) + + assert result.data["publishEvent"]["success"] is False + event.refresh_from_db() + assert event.status == EventStatus.DRAFT + + def test_publish_accepts_a_complete_event(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a) + + result = run(PUBLISH, ctx(org_a_admin, org_a), {"id": str(event.id)}) + + assert result.errors is None + assert result.data["publishEvent"]["success"] is True + assert result.data["publishEvent"]["data"]["status"] == "PUBLISHED" + + def test_listing_is_org_scoped(self, org_a_admin, org_b_admin, org_a, org_b): + _make_event(org_a_admin, org_a) + _make_event(org_b_admin, org_b, name="Other org") + + result = run(TABLE, ctx(org_a_admin, org_a)) + + assert result.errors is None + names = [row["name"] for row in result.data["eventsTable"]["data"]] + assert names == ["Existing meetup"] + + def test_anonymous_listing_is_published_only(self, org_a_admin, org_a): + _make_event(org_a_admin, org_a, name="Draft night") + _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + + result = run(TABLE, ctx(AnonymousUser())) + + statuses = [row["status"] for row in result.data["eventsTable"]["data"]] + assert statuses == ["PUBLISHED"] + + def test_published_listing_is_public_for_a_signed_in_user( + self, org_a_admin, org_b_admin, org_a, org_b, individual + ): + _make_event(org_a_admin, org_a, name="Draft night") + _make_event( + org_a_admin, + org_a, + name="Public night", + status=EventStatus.PUBLISHED, + start_date=date(2026, 6, 2), + ) + _make_event( + org_b_admin, + org_b, + name="Other public", + status=EventStatus.PUBLISHED, + start_date=date(2026, 6, 1), + ) + + result = run(PUBLISHED, ctx(individual)) + + assert result.errors is None + names = [row["name"] for row in result.data["publishedEvents"]["data"]] + assert names == ["Other public", "Public night"] + assert result.data["publishedEvents"]["totalItemsCount"] == 2 + + def test_published_listing_can_be_limited_to_one_organization( + self, org_a_admin, org_b_admin, org_a, org_b + ): + _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + _make_event(org_b_admin, org_b, name="Other public", status=EventStatus.PUBLISHED) + + result = run(PUBLISHED, ctx(AnonymousUser()), {"organizationSlug": "org-a"}) + + names = [row["name"] for row in result.data["publishedEvents"]["data"]] + assert names == ["Public night"] + + def test_event_by_slug_returns_a_published_event(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, name="Public night", status=EventStatus.PUBLISHED) + + result = run(BY_SLUG, ctx(AnonymousUser()), {"publicId": event.url_slug}) + + assert result.errors is None + assert result.data["eventBySlug"]["urlSlug"] == event.url_slug + assert result.data["eventBySlug"]["lumaEmbed"]["src"] == EMBED + + def test_event_by_slug_hides_drafts(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a, name="Draft night") + + result = run(BY_SLUG, ctx(AnonymousUser()), {"publicId": event.url_slug}) + + assert result.errors is None + assert result.data["eventBySlug"] is None + + def test_anonymous_cannot_read_a_draft(self, org_a_admin, org_a): + event = _make_event(org_a_admin, org_a) + + result = run(GET, ctx(AnonymousUser()), {"id": str(event.id)}) + + assert result.errors is not None + + def test_other_org_cannot_update(self, org_a_admin, org_b_admin, org_a, org_b): + event = _make_event(org_a_admin, org_a) + + result = run( + UPDATE, + ctx(org_b_admin, org_b), + {"input": {"id": str(event.id), "name": "Taken"}}, + ) + + assert result.data["updateEvent"]["success"] is False + event.refresh_from_db() + assert event.name == "Existing meetup" diff --git a/tests/schema/test_module_public_links.py b/tests/schema/test_module_public_links.py new file mode 100644 index 00000000..528191a8 --- /dev/null +++ b/tests/schema/test_module_public_links.py @@ -0,0 +1,215 @@ +import types + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.core.exceptions import ValidationError +from django.db.models import QuerySet + +from api.models import AIModel, Collaborative, Publication, UseCase +from api.schema.schema import schema +from api.utils.enums import CollaborativeStatus, PublicationStatus, UseCaseStatus +from authorization.models import User + + +def run(query, variables, user=None): + ctx = types.SimpleNamespace(user=user or AnonymousUser(), context={}) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def owner(db): + return User.objects.create(username="owner", keycloak_id="owner") + + +def make_model(owner, **kw): + data = dict( + name="sarvam-translate", + display_name="Sarvam Translate", + description="d", + model_type="TRANSLATION", + provider="OPENAI", + user=owner, + ) + data.update(kw) + return AIModel.objects.create(**data) + + +# --- every module: id, link, same title twice, rename keeps id --------------- + + +@pytest.mark.parametrize("kind", ["usecase", "collaborative", "publication", "aimodel"]) +def test_ids_links_and_duplicate_titles(kind, owner) -> None: + def make(title): + if kind == "usecase": + return UseCase.objects.create(title=title, user=owner) + if kind == "collaborative": + return Collaborative.objects.create(title=title, user=owner) + if kind == "publication": + return Publication.objects.create(title=title, user=owner) + return make_model(owner, display_name=title, name=f"m-{AIModel.objects.count()}") + + a = make("Air Quality Study") + b = make("Air Quality Study") + assert len(a.public_id) == 12 and a.public_id != b.public_id + assert a.url_slug == f"air-quality-study-{a.public_id}" + assert a.url_slug != b.url_slug + + old = a.public_id + if kind == "aimodel": + a.display_name = "Air Quality Study v2" + else: + a.title = "Air Quality Study v2" + a.save() + a.refresh_from_db() + assert a.public_id == old + assert a.url_slug == f"air-quality-study-v2-{old}" + + +def test_long_collaborative_title_saves(owner) -> None: + # full_clean checks slug length and format; the cut must not leave a trailing hyphen. + title = "District wise monthly rainfall and temperature readings for all states 1990" + c = Collaborative.objects.create(title=title, user=owner) + assert len(c.slug) <= 75 and not c.slug.endswith("-") + + +# --- use cases ----------------------------------------------------------------- + +UC = "query($s: String!) { useCaseBySlug(publicId: $s) { id publicId urlSlug } }" + + +def test_use_case_by_link(owner) -> None: + uc = UseCase.objects.create(title="Flood Maps", user=owner, status=UseCaseStatus.PUBLISHED) + for value in [uc.url_slug, uc.public_id, f"old-words-{uc.public_id}"]: + res = run(UC, {"s": value}) + assert res.errors is None, res.errors + assert res.data["useCaseBySlug"]["id"] == str(uc.id) + assert res.data["useCaseBySlug"]["urlSlug"] == uc.url_slug + + +def test_use_case_by_link_hides_drafts_and_plain_words(owner) -> None: + draft = UseCase.objects.create(title="Secret", user=owner) + pub = UseCase.objects.create(title="Open One", user=owner, status=UseCaseStatus.PUBLISHED) + for value in [draft.url_slug, pub.slug, "aaaaaaaaaaaa", str(pub.id)]: + res = run(UC, {"s": value}) + assert res.errors is None, res.errors + assert res.data["useCaseBySlug"] is None + + +# --- collaboratives ------------------------------------------------------------ + +CB = "query($s: String!) { collaborativeBySlug(slug: $s) { id publicId urlSlug } }" +CB_ID = "query($s: String!) { collaborativeBySlug(publicId: $s) { id publicId urlSlug } }" + + +def test_collaborative_by_link_and_old_slug(owner) -> None: + c = Collaborative.objects.create( + title="Clean Rivers", user=owner, status=CollaborativeStatus.PUBLISHED + ) + for value in [c.url_slug, c.public_id, f"old-words-{c.public_id}", c.slug]: + res = run(CB, {"s": value}) + assert res.errors is None, res.errors + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + for value in [c.url_slug, c.public_id, f"old-words-{c.public_id}"]: + res = run(CB_ID, {"s": value}) + assert res.errors is None, res.errors + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + + +def test_collaborative_public_id_does_not_fall_back_to_slug(owner) -> None: + c = Collaborative.objects.create(title="Clean Rivers", user=owner) + res = run(CB_ID, {"s": c.slug}) + assert res.errors + + +def test_collaborative_same_title_gets_its_own_slug(owner) -> None: + # The slug is also the subdomain, so a repeated title gets a suffix. + first = Collaborative.objects.create(title="Clean Rivers", user=owner) + second = Collaborative.objects.create(title="Clean Rivers", user=owner) + assert first.slug == "clean-rivers" + assert second.slug.startswith("clean-rivers-") and len(second.slug) == len("clean-rivers") + 5 + for c in [first, second]: + res = run(CB, {"s": c.slug}) + assert res.data["collaborativeBySlug"]["id"] == str(c.id) + + +def test_collaborative_taken_slug_is_refused(owner) -> None: + Collaborative.objects.create(title="Clean Rivers", user=owner) + other = Collaborative.objects.create(title="Something else", user=owner) + other.slug = "clean-rivers" + with pytest.raises(ValidationError): + other.save() + + +def test_collaborative_slug_race_gets_a_new_suffix(owner, monkeypatch) -> None: + # Another save takes the slug between the check and the insert. + Collaborative.objects.create(title="Clean Rivers", user=owner) + monkeypatch.setattr(Collaborative, "full_clean", lambda self, **kw: None) + real_exists = QuerySet.exists + monkeypatch.setattr(QuerySet, "exists", lambda self: False) + c = Collaborative.objects.create(title="Clean Rivers", user=owner) + monkeypatch.setattr(QuerySet, "exists", real_exists) + assert c.slug.startswith("clean-rivers-") + + +def test_collaborative_unknown_link_errors(owner) -> None: + res = run(CB, {"s": "nothing-aaaaaaaaaaaa"}) + assert res.errors + + +# --- publications -------------------------------------------------------------- + +PB = """query($id: UUID, $p: String) { + getPublication(publicationId: $id, publicId: $p) { id publicId urlSlug } +}""" + + +def test_publication_by_public_id(owner) -> None: + p = Publication.objects.create( + title="Budget Brief", user=owner, status=PublicationStatus.PUBLISHED + ) + for value in [p.url_slug, p.public_id]: + res = run(PB, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getPublication"]["id"] == str(p.id) + res = run(PB, {"id": str(p.id)}) + assert res.data["getPublication"]["publicId"] == p.public_id + + +def test_publication_draft_by_public_id(owner) -> None: + p = Publication.objects.create(title="Draft Brief", user=owner) + assert run(PB, {"p": p.public_id}).errors # anonymous refused + stranger = User.objects.create(username="s", keycloak_id="s") + assert run(PB, {"p": p.public_id}, stranger).errors + res = run(PB, {"p": p.public_id}, owner) + assert res.errors is None and res.data["getPublication"]["id"] == str(p.id) + + +def test_publication_unknown_gives_null(owner) -> None: + for value in ["aaaaaaaaaaaa", "no-id-here"]: + res = run(PB, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getPublication"] is None + + +# --- AI models ----------------------------------------------------------------- + +AM = """query($id: Int, $p: String) { + getAiModel(modelId: $id, publicId: $p) { id publicId urlSlug } +}""" + + +def test_ai_model_by_public_id(owner) -> None: + m = make_model(owner, is_public=True, is_active=True) + for value in [m.url_slug, m.public_id]: + res = run(AM, {"p": value}) + assert res.errors is None, res.errors + assert res.data["getAiModel"]["id"] == m.id + assert run(AM, {"id": m.id}).data["getAiModel"]["publicId"] == m.public_id + + +def test_ai_model_private_and_unknown(owner) -> None: + m = make_model(owner, is_public=False, is_active=True) + assert run(AM, {"p": m.public_id}).data["getAiModel"] is None + assert run(AM, {"p": m.public_id}, owner).data["getAiModel"]["id"] == m.id + assert run(AM, {"p": "aaaaaaaaaaaa"}).data["getAiModel"] is None + assert run(AM, {"p": "words-only"}).data["getAiModel"] is None diff --git a/tests/schema/test_resource_permissions.py b/tests/schema/test_resource_permissions.py new file mode 100644 index 00000000..72e86a66 --- /dev/null +++ b/tests/schema/test_resource_permissions.py @@ -0,0 +1,183 @@ +"""Resource mutations need edit rights on the dataset; drafts' resources are +visible only to people who can see the draft.""" + +import types +import uuid + +import pytest +from django.contrib.auth.models import AnonymousUser +from django.test import RequestFactory + +from api.models import Dataset, Organization, Resource +from api.schema.schema import schema +from authorization.models import DatasetPermission, OrganizationMembership, Role, User + +DENIED = "You don't have permission to change this dataset's resources" + + +def run(user, query, **variables): + request = RequestFactory().post("/api/graphql") + request.user = user + request.context = {} + ctx = types.SimpleNamespace(request=request, user=user, context={}, response=None) + return schema.execute_sync(query, variable_values=variables, context_value=ctx) + + +@pytest.fixture +def world(db): + org = Organization.objects.create(name="Org", description="o", slug="org") + other_org = Organization.objects.create(name="Other", description="o", slug="other-org") + editor_role = Role.objects.create(name="editor", can_view=True, can_add=True, can_change=True) + viewer_role = Role.objects.create(name="viewer", can_view=True) + users = { + name: User.objects.create(username=name, keycloak_id=name) + for name in ["owner", "editor", "viewer", "outsider", "shared"] + } + users["admin"] = User.objects.create(username="admin", keycloak_id="admin", is_superuser=True) + OrganizationMembership.objects.create(user=users["editor"], organization=org, role=editor_role) + OrganizationMembership.objects.create(user=users["viewer"], organization=org, role=viewer_role) + OrganizationMembership.objects.create( + user=users["outsider"], organization=other_org, role=editor_role + ) + draft = Dataset.objects.create( + title="Draft", slug="draft", description="d", user=users["owner"], organization=org + ) + published = Dataset.objects.create( + title="Pub", + slug="pub", + description="d", + user=users["owner"], + organization=org, + status="PUBLISHED", + ) + DatasetPermission.objects.create(user=users["shared"], dataset=draft, role=editor_role) + return types.SimpleNamespace( + users=users, + draft=draft, + published=published, + draft_res=Resource.objects.create(name="d.csv", dataset=draft), + pub_res=Resource.objects.create(name="p.csv", dataset=published), + ) + + +DELETE = "mutation ($id: UUID!) { deleteFileResource(resourceId: $id) }" +RENAME = """mutation ($id: UUID!) { + updateFileResource(fileResourceInput: {id: $id, name: "renamed"}) { ... on TypeResource { name } } +}""" +CREATE_EMPTY = """mutation ($ds: UUID!) { + createFileResource(fileResourceInput: {dataset: $ds}) { ... on TypeResource { id } } +}""" +CREATE_FILES = """mutation ($ds: UUID!) { + createFileResources(fileResourceInput: {dataset: $ds, files: []}) { id } +}""" +RESET_SCHEMA = """mutation ($id: UUID!) { + resetFileResourceSchema(resourceId: $id) { ... on TypeResource { id } } +}""" +UPDATE_SCHEMA = """mutation ($id: UUID!) { + updateFileResourceSchema(schemaUpdateInput: {resource: $id, updates: []}) { + ... on TypeResource { id } + } +}""" +MAJOR = """mutation ($id: UUID!) { + createMajorVersion(input: {resourceId: $id, description: "x"}) { ... on TypeResource { id } } +}""" + + +def denied(result): + """Either a top-level GraphQL error or a handled OperationInfo, both with DENIED.""" + if result.errors: + return any(DENIED in str(e.message) for e in result.errors) + payload = next(iter(result.data.values())) + return bool(payload) and DENIED in str(payload.get("messages", payload)) + + +@pytest.mark.parametrize("who", ["anonymous", "outsider", "viewer"]) +@pytest.mark.parametrize( + "query,arg", + [ + (DELETE, "res"), + (RENAME, "res"), + (RESET_SCHEMA, "res"), + (UPDATE_SCHEMA, "res"), + (MAJOR, "res"), + (CREATE_EMPTY, "ds"), + (CREATE_FILES, "ds"), + ], +) +def test_people_without_edit_rights_are_refused(world, who, query, arg): + user = AnonymousUser() if who == "anonymous" else world.users[who] + variables = {"id": str(world.pub_res.id)} if arg == "res" else {"ds": str(world.published.id)} + if query in (RENAME, UPDATE_SCHEMA, RESET_SCHEMA, CREATE_EMPTY, MAJOR): + query = query.replace("{ ... on TypeResource", "{ __typename ... on TypeResource") + query = query.replace( + "{ id } }\n}", "{ id } ... on OperationInfo { messages { message } } }\n}" + ) + query = query.replace( + "{ name } }\n}", "{ name } ... on OperationInfo { messages { message } } }\n}" + ) + r = run(user, query, **variables) + assert denied(r), (who, r.errors, r.data) + assert Resource.objects.filter(id=world.pub_res.id, name="p.csv").exists() + + +@pytest.mark.parametrize("who", ["owner", "editor", "admin", "shared"]) +def test_people_with_edit_rights_can_rename_and_delete(world, who): + user = world.users[who] + r = run(user, RENAME, id=str(world.draft_res.id)) + assert r.errors is None, r.errors + assert r.data["updateFileResource"]["name"] == "renamed" + + r = run(user, DELETE, id=str(world.draft_res.id)) + assert r.errors is None and r.data["deleteFileResource"] is True + assert not Resource.objects.filter(id=world.draft_res.id).exists() + + +def test_shared_editor_loses_rights_once_published(world): + # Dataset-level roles only cover drafts, as for dataset edits. + r = run(world.users["shared"], DELETE, id=str(world.pub_res.id)) + assert denied(r) + + +def test_missing_id_is_refused_like_a_forbidden_one(world): + r = run(AnonymousUser(), DELETE, id=str(uuid.uuid4())) + assert denied(r) + assert "does not exist" not in str(r.errors) + + +LIST = "query ($ds: UUID!) { datasetResources(datasetId: $ds) { name } }" +ONE = "query ($id: UUID!) { resourceById(resourceId: $id) { name } }" + + +def test_published_resources_stay_public(world): + r = run(AnonymousUser(), LIST, ds=str(world.published.id)) + assert r.errors is None and r.data["datasetResources"] == [{"name": "p.csv"}] + r = run(AnonymousUser(), ONE, id=str(world.pub_res.id)) + assert r.errors is None and r.data["resourceById"]["name"] == "p.csv" + + +@pytest.mark.parametrize("who", ["anonymous", "outsider"]) +def test_draft_resources_are_hidden(world, who): + user = AnonymousUser() if who == "anonymous" else world.users[who] + r = run(user, LIST, ds=str(world.draft.id)) + assert r.errors is None and r.data["datasetResources"] == [] + r = run(user, ONE, id=str(world.draft_res.id)) + assert r.errors and "does not exist" in str(r.errors[0].message) + + +@pytest.mark.parametrize("who", ["owner", "editor", "viewer", "shared", "admin"]) +def test_draft_resources_visible_to_people_who_can_see_the_draft(world, who): + user = world.users[who] + r = run(user, LIST, ds=str(world.draft.id)) + assert r.errors is None and r.data["datasetResources"] == [{"name": "d.csv"}] + r = run(user, ONE, id=str(world.draft_res.id)) + assert r.errors is None and r.data["resourceById"]["name"] == "d.csv" + + +def test_all_resources_list_hides_drafts(world): + query = "{ resources { name } }" + r = run(AnonymousUser(), query) + assert r.errors is None and r.data["resources"] == [{"name": "p.csv"}] + r = run(world.users["outsider"], query) + assert r.data["resources"] == [{"name": "p.csv"}] + r = run(world.users["owner"], query) + assert sorted(x["name"] for x in r.data["resources"]) == ["d.csv", "p.csv"] diff --git a/tests/test_data_indexing_filters.py b/tests/test_data_indexing_filters.py new file mode 100644 index 00000000..06d71dd9 --- /dev/null +++ b/tests/test_data_indexing_filters.py @@ -0,0 +1,203 @@ +"""Unit tests for the SQL-builder / filter helpers in +``api.utils.data_indexing`` and the request-parsing helpers in +``api.views.dataset_data``. + +These tests deliberately avoid touching the actual ``data_db`` connection — +they validate the logic that turns user input into safe SQL fragments and +into normalised filter dicts. +""" + +import unittest + +from django.http import QueryDict +from psycopg2 import sql as pg_sql + +from api.utils.data_indexing import ( + DataFetchError, + _build_order_by, + _build_select_list, + _build_where_clause, + _parse_filter_key, +) +from api.views.dataset_data import _extract_filters, _parse_bool, _parse_int + + +def _render(composable: pg_sql.Composable) -> str: + """Stringify a Composable without needing a live DB connection. + + Walks the Composed tree and concatenates the literal strings of each + leaf (SQL/Identifier). Identifiers are rendered as ``"name"``. + """ + if isinstance(composable, pg_sql.SQL): + return composable.string + if isinstance(composable, pg_sql.Identifier): + # psycopg2 may store multiple components for schema-qualified idents + parts = ( + composable.strings + if hasattr(composable, "strings") + else (composable._wrapped if hasattr(composable, "_wrapped") else []) + ) + return ".".join(f'"{p}"' for p in parts) + if isinstance(composable, pg_sql.Composed): + return "".join(_render(c) for c in composable.seq) + if isinstance(composable, pg_sql.Placeholder): + return "%s" + return str(composable) + + +class TestParseFilterKey(unittest.TestCase): + def test_no_op_defaults_to_eq(self) -> None: + self.assertEqual(_parse_filter_key("price"), ("price", "eq")) + + def test_known_op_split(self) -> None: + self.assertEqual(_parse_filter_key("price__gte"), ("price", "gte")) + self.assertEqual(_parse_filter_key("name__icontains"), ("name", "icontains")) + + def test_unknown_op_treated_as_column(self) -> None: + # Column may legitimately contain "__" — if suffix isn't a known op, + # fall back to equality on the full key. + col, op = _parse_filter_key("weird__suffix") + self.assertEqual((col, op), ("weird__suffix", "eq")) + + +class TestBuildWhereClause(unittest.TestCase): + allowed = ["id", "price", "name", "active"] + + def test_empty_filters(self) -> None: + sql, params = _build_where_clause({}, self.allowed) + self.assertEqual(params, []) + self.assertEqual(_render(sql), "") + + def test_eq_and_gte(self) -> None: + sql, params = _build_where_clause({"price__gte": 10, "name": "abc"}, self.allowed) + rendered = _render(sql) + self.assertIn(" WHERE ", rendered) + self.assertIn('"price" >= %s', rendered) + self.assertIn('"name" = %s', rendered) + self.assertIn(10, params) + self.assertIn("abc", params) + + def test_in_operator_normalises_to_list(self) -> None: + sql, params = _build_where_clause({"id__in": ("a", "b")}, self.allowed) + rendered = _render(sql) + self.assertIn("= ANY(%s)", rendered) + self.assertEqual(params, [["a", "b"]]) + + def test_isnull_truthy(self) -> None: + sql, params = _build_where_clause({"name__isnull": True}, self.allowed) + rendered = _render(sql) + self.assertIn("IS NULL", rendered) + self.assertEqual(params, []) + + def test_isnull_false_means_not_null(self) -> None: + sql, _ = _build_where_clause({"name__isnull": "false"}, self.allowed) + self.assertIn("IS NOT NULL", _render(sql)) + + def test_unknown_column_rejected(self) -> None: + with self.assertRaises(DataFetchError): + _build_where_clause({"evil__gte": 1}, self.allowed) + + def test_icontains_wraps_value(self) -> None: + _, params = _build_where_clause({"name__icontains": "foo"}, self.allowed) + self.assertEqual(params, ["%foo%"]) + + def test_startswith_wraps_value(self) -> None: + _, params = _build_where_clause({"name__startswith": "foo"}, self.allowed) + self.assertEqual(params, ["foo%"]) + + +class TestBuildOrderBy(unittest.TestCase): + allowed = ["id", "price"] + + def test_none_returns_empty(self) -> None: + sql = _build_order_by(None, self.allowed) + self.assertEqual(_render(sql), "") + + def test_asc_and_desc(self) -> None: + sql = _build_order_by(["-price", "id"], self.allowed) + rendered = _render(sql) + self.assertIn(" ORDER BY ", rendered) + self.assertIn('"price" DESC', rendered) + self.assertIn('"id" ASC', rendered) + + def test_unknown_column_rejected(self) -> None: + with self.assertRaises(DataFetchError): + _build_order_by(["evil"], self.allowed) + + +def _fill_params(query: str, params: list) -> str: + """Fill in ``%s`` placeholders the way psycopg2 does before sending a query. + + psycopg2 runs the whole query text through %-formatting whenever a + parameter list is passed (even an empty one), so a lone ``%`` anywhere in + the text, including inside a quoted column name, breaks it. + """ + return query % tuple(f"'{p}'" for p in params) + + +class TestColumnNamesWithPercentSign(unittest.TestCase): + """Indicator tables often have headers like "Literacy rate (%)".""" + + column = "Prevalence of Anaemia Among Women of Age 15 to 49 (%)" + + def test_select_list_keeps_the_column_name_after_params_are_filled(self) -> None: + sql = _build_select_list([self.column, "District"]) + self.assertEqual(_fill_params(_render(sql), []), f'"{self.column}", "District"') + + def test_filter_keeps_the_column_name_after_params_are_filled(self) -> None: + sql, params = _build_where_clause({self.column: "Kamrup"}, [self.column]) + self.assertEqual(_fill_params(_render(sql), params), f" WHERE \"{self.column}\" = 'Kamrup'") + + def test_order_by_keeps_the_column_name_after_params_are_filled(self) -> None: + sql = _build_order_by([f"-{self.column}"], [self.column]) + self.assertEqual(_fill_params(_render(sql), []), f' ORDER BY "{self.column}" DESC') + + +class TestViewQueryParamHelpers(unittest.TestCase): + def test_parse_bool(self) -> None: + self.assertTrue(_parse_bool("true")) + self.assertTrue(_parse_bool("YES")) + self.assertTrue(_parse_bool(True)) + self.assertFalse(_parse_bool("0")) + self.assertFalse(_parse_bool(None, default=False)) + self.assertTrue(_parse_bool(None, default=True)) + + def test_parse_int(self) -> None: + self.assertEqual(_parse_int("42", 0), 42) + self.assertEqual(_parse_int(None, 7), 7) + self.assertEqual(_parse_int("not-a-number", 9), 9) + + def test_extract_filters_skips_reserved(self) -> None: + qd = QueryDict(mutable=True) + qd.update({"limit": "10", "offset": "0", "columns": "a,b"}) + qd["price__gte"] = "5" + qd["name"] = "abc" + result = _extract_filters(qd) + self.assertEqual(result, {"price__gte": "5", "name": "abc"}) + + def test_extract_filters_in_collapses_to_list(self) -> None: + qd = QueryDict("col__in=a&col__in=b&col__in=c,d") + result = _extract_filters(qd) + self.assertIn("col__in", result) + self.assertEqual(sorted(result["col__in"]), ["a", "b", "c", "d"]) + + def test_extract_filters_custom_reserved(self) -> None: + qd = QueryDict("limit=10&prompt_contains=x&col=y") + result = _extract_filters( + qd, + reserved={ + "limit", + "offset", + "columns", + "order_by", + "count", + "resource_id", + "format", + "prompt_contains", + }, + ) + self.assertEqual(result, {"col": "y"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 74653975..7155fa8e 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -140,5 +140,117 @@ def test_search_with_sorting(self, mock_request: MagicMock) -> None: mock_request.assert_called_once() +class TestDatasetClientDataFetch(unittest.TestCase): + """Tests for indexed-data fetch methods on DatasetClient.""" + + def setUp(self) -> None: + self.client = DatasetClient("https://api.test.com", MagicMock()) + + def test_build_data_params_basic(self) -> None: + params = DatasetClient._build_data_params( + filters=None, + columns=None, + order_by=None, + limit=50, + offset=10, + count=True, + ) + self.assertEqual(params["limit"], 50) + self.assertEqual(params["offset"], 10) + self.assertEqual(params["count"], "true") + self.assertNotIn("columns", params) + + def test_build_data_params_filters_and_lists(self) -> None: + params = DatasetClient._build_data_params( + filters={"price__gte": 10, "tag__in": ["a", "b"], "active": True}, + columns=["id", "name"], + order_by=["-price", "name"], + limit=100, + offset=0, + count=False, + ) + self.assertEqual(params["columns"], "id,name") + self.assertEqual(params["order_by"], "-price,name") + self.assertEqual(params["count"], "false") + self.assertEqual(params["price__gte"], 10) + self.assertEqual(params["tag__in"], ["a", "b"]) + self.assertEqual(params["active"], "true") + + @patch.object(DatasetClient, "get") + def test_get_resource_data(self, mock_get: MagicMock) -> None: + mock_get.return_value = { + "columns": ["id"], + "rows": [[1]], + "total": 1, + "limit": 100, + "offset": 0, + } + result = self.client.get_resource_data( + "res-1", + filters={"id__gte": 1}, + columns=["id"], + order_by=["id"], + limit=10, + ) + self.assertEqual(result["total"], 1) + endpoint, kwargs = mock_get.call_args[0][0], mock_get.call_args.kwargs + self.assertEqual(endpoint, "/api/resources/res-1/data/") + self.assertEqual(kwargs["params"]["columns"], "id") + self.assertEqual(kwargs["params"]["id__gte"], 1) + + @patch.object(DatasetClient, "get") + def test_get_dataset_data_with_resource_id(self, mock_get: MagicMock) -> None: + mock_get.return_value = {"rows": [], "columns": [], "total": 0} + self.client.get_dataset_data("ds-1", resource_id="res-9", limit=5) + endpoint = mock_get.call_args[0][0] + params = mock_get.call_args.kwargs["params"] + self.assertEqual(endpoint, "/api/datasets/ds-1/data/") + self.assertEqual(params["resource_id"], "res-9") + self.assertEqual(params["limit"], 5) + + @patch.object(DatasetClient, "get") + def test_get_prompt_data_shorthands(self, mock_get: MagicMock) -> None: + mock_get.return_value = {"rows": [], "columns": [], "total": 0} + self.client.get_prompt_data( + "ds-1", + prompt_contains="translate", + response_contains="hello", + min_length=5, + max_length=100, + ) + endpoint = mock_get.call_args[0][0] + params = mock_get.call_args.kwargs["params"] + self.assertEqual(endpoint, "/api/datasets/ds-1/prompts/") + self.assertEqual(params["prompt_contains"], "translate") + self.assertEqual(params["response_contains"], "hello") + self.assertEqual(params["min_length"], 5) + self.assertEqual(params["max_length"], 100) + + @patch.object(DatasetClient, "get_resource_data") + def test_iter_resource_data_paginates(self, mock_get_data: MagicMock) -> None: + # Two pages: full batch then partial page (terminator) + mock_get_data.side_effect = [ + {"columns": ["id", "name"], "rows": [[1, "a"], [2, "b"]]}, + {"columns": ["id", "name"], "rows": [[3, "c"]]}, + ] + rows = list(self.client.iter_resource_data("res-1", batch_size=2)) + self.assertEqual( + rows, + [ + {"id": 1, "name": "a"}, + {"id": 2, "name": "b"}, + {"id": 3, "name": "c"}, + ], + ) + self.assertEqual(mock_get_data.call_count, 2) + # Second call advances offset + self.assertEqual(mock_get_data.call_args_list[1].kwargs["offset"], 2) + + @patch.object(DatasetClient, "get_resource_data") + def test_iter_resource_data_empty(self, mock_get_data: MagicMock) -> None: + mock_get_data.return_value = {"columns": ["id"], "rows": []} + self.assertEqual(list(self.client.iter_resource_data("res-1")), []) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_license_list.py b/tests/test_license_list.py new file mode 100644 index 00000000..2d3136c7 --- /dev/null +++ b/tests/test_license_list.py @@ -0,0 +1,123 @@ +"""DatasetLicense offers one choice per row of contracts/licenses.csv. + +The CSV is the source of truth: its ``dataspace_enum`` alt code names the enum +value, its label is the choice label, and its URI is what the metadata export +emits for that value. +""" + +import csv +import re +from pathlib import Path + +import pytest +from django.core.exceptions import ValidationError + +from api.models import Dataset, Publication +from api.schema.schema import schema +from api.services.metadata_export import vocabularies as vocab +from api.services.metadata_export.exporter import export_dataset +from api.services.publication_service import validate_draft_inputs +from api.utils.enums import DatasetLicense +from tests.schema.test_publication_schema import ( # noqa: F401 (fixtures) + CREATE, + _no_activity_recording, + ctx, + geography, + org_a, + org_a_admin, + resource_type, + roles, + run, + sector, + valid_create_vars, +) + +LICENSES_CSV = ( + Path(__file__).resolve().parent.parent / "api/services/metadata_export/contracts/licenses.csv" +) + + +def _rows(): + with open(LICENSES_CSV, encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + +ROWS = _rows() + + +def _enum_value(row): + match = re.search(r"dataspace_enum=([A-Z0-9_]+)", row["alt_codes"]) + assert match, f"{row['key']} has no dataspace_enum alt code" + return match.group(1) + + +class TestEnumMatchesCsv: + def test_one_choice_per_row(self): + assert sorted(DatasetLicense.values) == sorted(_enum_value(r) for r in ROWS) + + def test_labels_come_from_the_csv(self): + labels = dict(DatasetLicense.choices) + for row in ROWS: + assert labels[_enum_value(row)] == " ".join(row["label"].split()), row["key"] + + def test_existing_values_are_unchanged(self): + # Stored rows use these; renaming one would orphan data. + for value in ( + "GOVERNMENT_OPEN_DATA_LICENSE", + "CC_BY_4_0_ATTRIBUTION", + "CC_BY_SA_4_0_ATTRIBUTION_SHARE_ALIKE", + "OPEN_DATA_COMMONS_BY_ATTRIBUTION", + "OPEN_DATABASE_LICENSE", + ): + assert value in DatasetLicense.values + + def test_values_fit_the_column(self): + field = Dataset._meta.get_field("license") + assert max(len(v) for v in DatasetLicense.values) <= field.max_length + + @pytest.mark.parametrize("row", ROWS, ids=lambda r: r["key"]) + def test_every_value_resolves_to_its_uri(self, row): + assert vocab.license_uri(_enum_value(row)) == row["uri"] + + +@pytest.mark.django_db +class TestStorageAndExport: + @pytest.mark.parametrize("value", ["CC_BY_NC_ND_4_0", "CC0_1_0", "CC_BY_4_0_ATTRIBUTION"]) + def test_dataset_exports_the_license_uri(self, value, org_a): # noqa: F811 + dataset = Dataset.objects.create(title="Rainfall", organization=org_a, license=value) + body, _, _, _ = export_dataset(dataset, "dcat", "jsonld") + assert vocab.license_uri(value) in body + + def test_value_outside_the_list_is_rejected(self, org_a): # noqa: F811 + dataset = Dataset(title="Rainfall", organization=org_a, license="MIT") + with pytest.raises(ValidationError) as exc: + dataset.full_clean(exclude=["slug", "user", "dataspace", "description"]) + assert "license" in exc.value.message_dict + + +class TestGraphQL: + def test_enum_exposes_every_value(self): + result = schema.execute_sync('{ __type(name: "DatasetLicense") { enumValues { name } } }') + assert result.errors is None + names = {v["name"] for v in result.data["__type"]["enumValues"]} + assert names == set(DatasetLicense.values) + + @pytest.mark.django_db + @pytest.mark.parametrize("value", ["CC_BY_NC_4_0", "PDDL_1_0"]) + def test_create_publication_accepts_new_values( + self, value, org_a_admin, org_a, resource_type, sector, geography # noqa: F811 + ): + result = run( + CREATE, + ctx(org_a_admin, org_a), + valid_create_vars(resource_type, sector, geography, license=value), + ) + assert result.errors is None + payload = result.data["createPublication"] + assert payload["success"] is True + assert Publication.objects.get(id=payload["data"]["id"]).license == value + + def test_service_validation_uses_the_full_list(self): + validate_draft_inputs(license_value="CC_BY_SA_3_0_IGO") + with pytest.raises(ValidationError): + validate_draft_inputs(license_value="NOT_A_LICENSE") diff --git a/tests/test_migration_graph.py b/tests/test_migration_graph.py new file mode 100644 index 00000000..114259e0 --- /dev/null +++ b/tests/test_migration_graph.py @@ -0,0 +1,26 @@ +"""Regression test: the migration graph must have a single leaf per app. + +Root cause: ``0056_file_field_storages`` (s3_migration) and ``0056_update_license_choices`` +(dev) both depended on ``0055_event``, and dev then added ``0057`` on top of its own branch. +Merging dev left two leaf nodes, so ``manage.py migrate`` aborted with "Conflicting migrations +detected" and the container never started. The suite did not notice because ``--nomigrations`` +hides the real migration modules. + +Fix: renumber the branch's migration after the latest one on dev and fix its ``dependencies``. +""" + +from django.db.migrations.loader import MigrationLoader +from django.test import override_settings + + +# pytest's --nomigrations disables MIGRATION_MODULES, which would leave an empty graph here and +# make these tests pass whatever is on disk, so restore the real modules. +@override_settings(MIGRATION_MODULES={}) +def test_migration_graph_has_no_conflicts() -> None: + assert MigrationLoader(None).detect_conflicts() == {} + + +@override_settings(MIGRATION_MODULES={}) +def test_migration_graph_is_actually_loaded() -> None: + # Guards the test above against passing on an empty graph. + assert MigrationLoader(None).graph.leaf_nodes("api") diff --git a/tests/test_platform_license_mapping.py b/tests/test_platform_license_mapping.py new file mode 100644 index 00000000..2338ab4c --- /dev/null +++ b/tests/test_platform_license_mapping.py @@ -0,0 +1,70 @@ +"""Imported datasets keep the license the platform reported. + +SPDX ids map straight onto DatasetLicense; a few platform-specific names go +through LICENSE_ALIASES; anything unknown falls back to CC-BY 4.0. +""" + +import csv +import re +from pathlib import Path + +import pytest + +from api.services.platform_importers.base import map_license +from api.services.platform_importers.github import GitHubImporter +from api.services.platform_importers.huggingface import HuggingFaceImporter +from api.services.platform_importers.kaggle import KaggleImporter + +LICENSES_CSV = ( + Path(__file__).resolve().parent.parent / "api/services/metadata_export/contracts/licenses.csv" +) + +with open(LICENSES_CSV, encoding="utf-8", newline="") as fh: + ROWS = list(csv.DictReader(fh)) + + +def _enum_value(row): + return re.search(r"dataspace_enum=([A-Z0-9_]+)", row["alt_codes"]).group(1) + + +@pytest.mark.parametrize("row", ROWS, ids=lambda r: r["key"]) +def test_spdx_code_maps_to_its_value(row): + assert map_license(row["code"]) == _enum_value(row) + assert map_license(row["code"].lower()) == _enum_value(row) + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("CC0: Public Domain", "CC0_1_0"), + ("CC BY-NC-SA 4.0", "CC_BY_NC_SA_4_0"), + ("Attribution 4.0 International (CC BY 4.0)", "CC_BY_4_0_ATTRIBUTION"), + ("ODbL", "OPEN_DATABASE_LICENSE"), + ("GODL-India", "GOVERNMENT_OPEN_DATA_LICENSE"), + ("MIT", "CC_BY_4_0_ATTRIBUTION"), + ("", "CC_BY_4_0_ATTRIBUTION"), + ], +) +def test_platform_names(raw, expected): + assert map_license(raw) == expected + + +def test_github_spdx_id(): + assert map_license(GitHubImporter._license({"spdx_id": "CC-BY-NC-4.0"})) == "CC_BY_NC_4_0" + assert map_license(GitHubImporter._license({"spdx_id": "NOASSERTION"})) == ( + "CC_BY_4_0_ATTRIBUTION" + ) + + +def test_huggingface_card_and_tag(): + assert map_license(HuggingFaceImporter._license({"license": "cc-by-nc-sa-4.0"}, [])) == ( + "CC_BY_NC_SA_4_0" + ) + assert map_license(HuggingFaceImporter._license({}, ["license:pddl"])) == "PDDL_1_0" + + +def test_kaggle_license_name(): + assert map_license(KaggleImporter._license({"licenses": [{"name": "CC0-1.0"}]})) == "CC0_1_0" + assert map_license(KaggleImporter._license({"licenseName": "ODC-By-1.0"})) == ( + "OPEN_DATA_COMMONS_BY_ATTRIBUTION" + ) diff --git a/tests/test_public_links.py b/tests/test_public_links.py new file mode 100644 index 00000000..9b287f3d --- /dev/null +++ b/tests/test_public_links.py @@ -0,0 +1,157 @@ +import uuid + +import pytest +from django.db import IntegrityError + +from api.utils.public_links import ( + PUBLIC_ID_LENGTH, + build_url_slug, + generate_public_id, + parse_public_id, + save_with_public_id, +) + + +class Record: + public_id = "" + + +def test_generate_public_id_format_and_randomness() -> None: + ids = {generate_public_id() for _ in range(20000)} + assert len(ids) == 20000 + for value in list(ids)[:500]: + assert len(value) == PUBLIC_ID_LENGTH + assert value.isascii() and (value.islower() or value.isdigit()) and value.isalnum() + + +def test_save_assigns_id_when_missing() -> None: + record = Record() + calls = [] + save_with_public_id(record, lambda: calls.append(record.public_id)) + assert len(calls) == 1 and calls[0] == record.public_id + assert parse_public_id(record.public_id) == record.public_id + + +def test_save_keeps_existing_id() -> None: + record = Record() + record.public_id = "k3x9m2qa7b1z" + save_with_public_id(record, lambda: None) + assert record.public_id == "k3x9m2qa7b1z" + + +def test_save_retries_with_new_id_on_clash() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(record.public_id) + if len(tried) < 3: + raise IntegrityError("UNIQUE constraint failed: dataset.public_id") + + save_with_public_id(record, save) + assert len(tried) == 3 and len(set(tried)) == 3 + assert record.public_id == tried[-1] + + +def test_save_postgres_style_error_is_retried() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(record.public_id) + if len(tried) == 1: + raise IntegrityError( + 'duplicate key value violates unique constraint "dataset_public_id_key"\n' + "DETAIL: Key (public_id)=(abc) already exists." + ) + + save_with_public_id(record, save) + assert len(tried) == 2 + + +def test_save_other_unique_errors_are_not_retried() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(1) + raise IntegrityError("UNIQUE constraint failed: dataset.slug") + + with pytest.raises(IntegrityError): + save_with_public_id(record, save) + assert len(tried) == 1 + + +def test_save_gives_up_after_max_attempts() -> None: + record = Record() + tried = [] + + def save() -> None: + tried.append(1) + raise IntegrityError("UNIQUE constraint failed: dataset.public_id") + + with pytest.raises(IntegrityError): + save_with_public_id(record, save, max_attempts=4) + assert len(tried) == 4 + + +@pytest.mark.parametrize( + "title, expected", + [ + ("City Air Quality Index", "city-air-quality-index-k3x9m2qa7b1z"), + (" Rainfall: 2020–2024 (Daily)! ", "rainfall-20202024-daily-k3x9m2qa7b1z"), + ("जनगणना डेटा 2011", "2011-k3x9m2qa7b1z"), + ("जनगणना डेटा", "k3x9m2qa7b1z"), + ("", "k3x9m2qa7b1z"), + (None, "k3x9m2qa7b1z"), + ("!!! ???", "k3x9m2qa7b1z"), + ], +) +def test_build_url_slug(title, expected) -> None: + assert build_url_slug(title, "k3x9m2qa7b1z") == expected + + +def test_build_url_slug_cuts_long_titles_between_words() -> None: + title = "District wise monthly rainfall and temperature readings for all states of India 1990 to 2024" + assert ( + build_url_slug(title, "k3x9m2qa7b1z") + == "district-wise-monthly-rainfall-and-temperature-k3x9m2qa7b1z" + ) + + +def test_build_url_slug_exact_limit_and_one_long_word() -> None: + exact = "a" * 20 + " " + "b" * 29 # 50 chars once slugified + assert build_url_slug(exact, "k3x9m2qa7b1z") == f"{'a' * 20}-{'b' * 29}-k3x9m2qa7b1z" + long_word = "x" * 80 + assert build_url_slug(long_word, "k3x9m2qa7b1z") == f"{'x' * 50}-k3x9m2qa7b1z" + + +@pytest.mark.parametrize( + "link, expected", + [ + ("city-air-quality-index-k3x9m2qa7b1z", "k3x9m2qa7b1z"), + ("k3x9m2qa7b1z", "k3x9m2qa7b1z"), + ("/city-air-quality-index-K3X9M2QA7B1Z/", "k3x9m2qa7b1z"), + ("old-words-that-changed-k3x9m2qa7b1z", "k3x9m2qa7b1z"), + (str(uuid.UUID("3f2a9c1e-7b4d-4e2a-9c1e-a1b2c3d4e5f6")), None), + ("3f2a9c1e7b4d4e2a9c1ea1b2c3d4e5f6", None), + ("123", None), + ("city-air-quality-index", None), + ("city-air-quality-k3x9m2qa7b1", None), + ("city-air-quality-k3x9m2qa7b1z9", None), + ("city-air-quality-k3x9m2qa7b1_", None), + ("city-air-quality-k3x9m2qa7b1z\n/", None), + ("k3x9m2qa7b1٣", None), + ("k3x9m2qa7b1é", None), + ("", None), + (None, None), + ], +) +def test_parse_public_id(link, expected) -> None: + assert parse_public_id(link) == expected + + +def test_build_then_parse_round_trip() -> None: + for title in ["City Air Quality Index", "जनगणना डेटा", "", "x" * 300]: + public_id = generate_public_id() + assert parse_public_id(build_url_slug(title, public_id)) == public_id diff --git a/tests/test_publication_blocks.py b/tests/test_publication_blocks.py index 6364830e..38ee1b98 100644 --- a/tests/test_publication_blocks.py +++ b/tests/test_publication_blocks.py @@ -1,6 +1,5 @@ """Layer 1/3/4 tests for content blocks: add/reorder/remove/replace + download gate.""" -import os from datetime import date import pytest @@ -24,11 +23,6 @@ VIDEO = "https://youtu.be/dQw4w9WgXcQ" -@pytest.fixture(autouse=True) -def media_root(settings, tmp_path): - settings.MEDIA_ROOT = str(tmp_path) - - @pytest.fixture def owner(db): return User.objects.create(username="owner", keycloak_id="owner") @@ -117,27 +111,27 @@ def test_reorder_sets_positions_to_requested_order(self, publication): class TestReplaceFile: def test_replace_swaps_file_and_removes_the_old_one(self, publication): block = add_file_block(publication, _pdf("first.pdf")) - old_path = block.file.path + storage, old_name = block.file.storage, block.file.name old_id = block.id - assert os.path.exists(old_path) + assert storage.exists(old_name) replace_block_file(block, _pdf("second.pdf")) assert block.id == old_id # same row, in-place assert block.file_name == "second.pdf" - assert not os.path.exists(old_path) # old file removed from disk + assert not storage.exists(old_name) # old file removed from storage @pytest.mark.django_db class TestDeleteRemovesFile: def test_deleting_a_block_removes_its_file(self, publication): block = add_file_block(publication, _pdf()) - path = block.file.path - assert os.path.exists(path) + storage, name = block.file.storage, block.file.name + assert storage.exists(name) block.delete() - assert not os.path.exists(path) # post_delete signal cleaned it up + assert not storage.exists(name) # post_delete signal cleaned it up # --------------------------------------------------------------------------- # diff --git a/tests/test_publication_models.py b/tests/test_publication_models.py index 6cc2209c..7e43d686 100644 --- a/tests/test_publication_models.py +++ b/tests/test_publication_models.py @@ -26,20 +26,23 @@ def org(db): @pytest.mark.django_db class TestPublicationSlug: - def test_two_same_title_publications_get_distinct_slugs(self, user): + def test_two_same_title_publications_get_distinct_links(self, user): + # Slugs may repeat; each publication is found by its own public_id. first = Publication.objects.create(title="Annual Report", user=user) second = Publication.objects.create(title="Annual Report", user=user) - assert first.slug == "annual-report" - assert second.slug == "annual-report-1" - assert first.slug != second.slug + assert first.slug == second.slug == "annual-report" + assert first.public_id != second.public_id + assert first.url_slug == f"annual-report-{first.public_id}" + assert first.url_slug != second.url_slug - def test_a_blank_title_still_gets_a_unique_slug(self, user): + def test_a_blank_title_still_gets_a_link(self, user): first = Publication.objects.create(title="", user=user) second = Publication.objects.create(title=" ", user=user) - assert first.slug == "untitled" - assert second.slug == "untitled-1" + assert first.slug == second.slug == "untitled" + assert first.url_slug == first.public_id + assert first.public_id != second.public_id def test_a_unicode_title_slugs_without_crashing(self, user): publication = Publication.objects.create(title="Report — Résumé 2024", user=user) diff --git a/tests/test_rebuild_long_text_indexes.py b/tests/test_rebuild_long_text_indexes.py index 1c35d6ad..0143409b 100644 --- a/tests/test_rebuild_long_text_indexes.py +++ b/tests/test_rebuild_long_text_indexes.py @@ -1,11 +1,11 @@ -"""Migration 0054 rebuilds only search indexes that still have a single-token copy.""" +"""Migration 0064 rebuilds only search indexes that still have a single-token copy.""" import importlib from unittest import mock from django.test import override_settings -migration = importlib.import_module("api.migrations.0054_rebuild_long_text_search_indexes") +migration = importlib.import_module("api.migrations.0064_rebuild_long_text_search_indexes") OLD = { "properties": { diff --git a/tests/test_s3_storage.py b/tests/test_s3_storage.py new file mode 100644 index 00000000..27c0ce58 --- /dev/null +++ b/tests/test_s3_storage.py @@ -0,0 +1,290 @@ +"""Regression tests for the move from local disk (and DVC) to S3-only media. + +Root cause: uploads lived on local disk under MEDIA_ROOT and several code paths +read them through ``FieldFile.path`` (data indexing, upload MIME validation, +search documents, DVC versioning). S3 has no local path, so each of those +would raise ``NotImplementedError`` once media moved to a bucket. DVC never +worked at all: files were never under ``DVC_REPO_PATH``, so every DVC call +raised and ``createMajorVersion`` always failed. + +Fix: media goes through two S3 storages (``public``/``private``, keys prefixed +by ``STORAGE_ENV``); readers that need a real file use ``local_copy()``; old +``/public/`` links redirect to public files only; DVC is removed and +``createMajorVersion`` writes the version row directly behind a permission +check. Stored names drop the ``files/public/``/``files/`` prefix left from the +disk layout (S3 keys already sit under ``/public``); ``migrate_media_to_s3`` +moves existing files to those names and repoints the rows. +""" + +import io +import types +from pathlib import Path +from unittest.mock import patch + +import pytest +from django.apps import apps +from django.core.files.base import ContentFile +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import models +from django.test import RequestFactory +from django.urls import resolve +from storages.backends.s3 import S3Storage + +from api.models import Dataset, Resource, ResourceFileDetails, ResourceVersion +from api.models.Organization import Organization +from api.schema.schema import schema +from api.utils.file_paths import ( + _chart_image_directory_path, + _organization_directory_path, + _publication_block_directory_path, +) +from api.utils.file_utils import check_mime_type, local_copy +from api.utils.storage import normalize_media_name, private_storage, public_storage +from api.views.legacy_media import legacy_media_redirect +from authorization.models import User +from DataSpace import settings as project_settings + +FAKE_KEYS = {"access_key": "AKIAEXAMPLEKEY000000", "secret_key": "not-a-real-secret"} + + +def _s3(options): + # URL building and presigning are local operations; no request reaches AWS. + return S3Storage(**{**options, **FAKE_KEYS}) + + +class TestS3StorageOptions: + def test_public_urls_are_unsigned_under_env_public_prefix(self): + url = _s3(project_settings._S3_PUBLIC_OPTIONS).url("organizations/a/logo.png") + assert url.endswith(f"/{project_settings.STORAGE_ENV}/public/organizations/a/logo.png") + assert "X-Amz-Signature" not in url + + def test_private_urls_are_presigned_for_five_minutes(self): + url = _s3(project_settings._S3_PRIVATE_OPTIONS).url("resources/data.csv") + assert f"/{project_settings.STORAGE_ENV}/private/resources/data.csv" in url + assert "X-Amz-Signature" in url + assert "X-Amz-Expires=300" in url + + def test_default_storage_is_private(self): + assert project_settings.STORAGES["default"] == project_settings.STORAGES["private"] + + def test_s3_has_no_local_path(self): + with pytest.raises(NotImplementedError): + _s3(project_settings._S3_PRIVATE_OPTIONS).path("resources/data.csv") + + +# A new file field that forgets storage= silently lands in the private default, +# so a public image would 403. Keep this list in step with the models. +PRIVATE_FILE_FIELDS = { + ("api", "resourcefiledetails", "file"), + ("api", "resourcechartimage", "image"), +} + + +class TestFileFieldStorages: + def _file_fields(self): + for model in apps.get_models(): + for field in model._meta.get_fields(): + if isinstance(field, models.FileField): + yield (model._meta.app_label, model._meta.model_name, field.name), field + + def test_every_file_field_names_public_or_private_storage(self): + for key, field in self._file_fields(): + assert field._storage_callable in (public_storage, private_storage), key + + def test_only_resource_files_and_chart_images_are_private(self): + private = { + key for key, field in self._file_fields() if field._storage_callable is private_storage + } + assert private == PRIVATE_FILE_FIELDS + + +@pytest.mark.django_db +class TestLocalCopy: + def _stored(self, name, content): + dataset = Dataset.objects.create(title="Local copy") + resource = Resource.objects.create(dataset=dataset, name="r", slug=f"r-{name}") + details = ResourceFileDetails(resource=resource, format="csv") + details.file.save(name, ContentFile(content), save=False) + return details.file + + def test_yields_a_local_file_with_the_stored_bytes_and_extension(self): + fieldfile = self._stored("data.csv", b"a,b\n1,2\n") + with local_copy(fieldfile) as path: + assert path.endswith(".csv") + assert Path(path).read_bytes() == b"a,b\n1,2\n" + + def test_removes_the_temp_file_even_when_the_caller_raises(self): + fieldfile = self._stored("boom.csv", b"x") + with pytest.raises(RuntimeError): + with local_copy(fieldfile) as path: + raise RuntimeError("boom") + assert not Path(path).exists() + + def test_mime_check_reads_a_stored_file_without_a_path(self): + fieldfile = self._stored("report.pdf", b"%PDF-1.7\n%\xe2\xe3\xcf\xd3\n") + assert check_mime_type(fieldfile) == "application/pdf" + + +class TestLegacyMediaRedirect: + def _get(self, name): + return legacy_media_redirect(RequestFactory().get(f"/public/{name}"), name) + + def test_route_points_at_the_redirect_view(self): + assert resolve("/public/files/public/x.png").func is legacy_media_redirect + + def test_public_file_redirects_to_its_storage_url(self): + name = public_storage().save("organizations/acme/png/logo.png", ContentFile(b"png")) + response = self._get(name) + assert response.status_code == 302 + assert response["Location"] == public_storage().url(name) + + def test_old_prefixed_link_redirects_to_the_renamed_file(self): + name = public_storage().save("organizations/old/png/logo.png", ContentFile(b"png")) + response = self._get(f"files/public/{name}") + assert response.status_code == 302 + assert response["Location"] == public_storage().url(name) + + def test_private_file_is_never_redirected(self): + name = private_storage().save("resources/secret.csv", ContentFile(b"a,b")) + assert self._get(name).status_code == 404 + + @pytest.mark.parametrize("name", ["../etc/passwd", "files/../../etc/passwd", "/etc/passwd"]) + def test_traversal_is_rejected(self, name): + assert self._get(name).status_code == 404 + + +MAJOR_VERSION = """ +mutation($input: CreateMajorVersionInput!) { + createMajorVersion(input: $input) { ... on TypeResource { id } } +} +""" + + +@pytest.mark.django_db +class TestCreateMajorVersionWithoutDvc: + @pytest.fixture + def owner(self): + return User.objects.create(username="owner", keycloak_id="owner") + + @pytest.fixture + def resource(self, owner): + dataset = Dataset.objects.create(title="Versioned", user=owner) + return Resource.objects.create(dataset=dataset, name="data", slug="data-versioned") + + def _run(self, user, resource): + return schema.execute_sync( + MAJOR_VERSION, + variable_values={"input": {"resourceId": str(resource.id), "description": "schema"}}, + context_value=types.SimpleNamespace(user=user, context={}), + ) + + def test_owner_gets_a_version_row_and_bump(self, owner, resource): + result = self._run(owner, resource) + assert result.errors is None, result.errors + resource.refresh_from_db() + assert resource.version == "v1.0.0" + assert ResourceVersion.objects.filter(resource=resource, version_number="v1.0.0").exists() + + def test_a_stranger_cannot_version_someone_elses_resource(self, resource): + stranger = User.objects.create(username="stranger", keycloak_id="stranger") + result = self._run(stranger, resource) + assert result.errors + assert not ResourceVersion.objects.filter(resource=resource).exists() + + +class TestNameLayout: + def test_upload_names_carry_no_legacy_prefix(self): + org = types.SimpleNamespace(name="Acme") + chart = types.SimpleNamespace(name="Rain") + block = types.SimpleNamespace(publication_id="p1") + assert _organization_directory_path(org, "logo.png") == "organizations/Acme/png/logo.png" + assert _chart_image_directory_path(chart, "c.png") == "chart_images/Rain/png/c.png" + assert _publication_block_directory_path(block, "r.pdf").startswith("publications/p1/") + + @pytest.mark.parametrize( + "old, new", + [ + ("files/public/organizations/a/png/l.png", "organizations/a/png/l.png"), + ("files/use_case/Health/logo/l.png", "use_case/Health/logo/l.png"), + ("resources/data.csv", "resources/data.csv"), + ], + ) + def test_normalize_media_name(self, old, new): + assert normalize_media_name(old) == new + + +@pytest.mark.django_db +class TestMigrateMediaToS3: + OLD = "files/public/organizations/acme/png/logo.png" + NEW = "organizations/acme/png/logo.png" + + @pytest.fixture + def storage(self): + storage = Organization._meta.get_field("logo").storage + for name in (self.OLD, self.NEW): + if storage.exists(name): + storage.delete(name) + return storage + + @pytest.fixture + def source(self, tmp_path): + logo = tmp_path / self.OLD + logo.parent.mkdir(parents=True) + logo.write_bytes(b"logo-bytes") + (tmp_path / "stray.txt").write_text("no row points here") + return tmp_path + + @pytest.fixture + def org(self, storage): + return Organization.objects.create(name="Acme", slug="acme", logo=self.OLD) + + def _run(self, *args): + out = io.StringIO() + call_command("migrate_media_to_s3", *args, stdout=out) + return out.getvalue() + + def _read(self, storage, name): + with storage.open(name) as fh: + return fh.read() + + def test_dry_run_writes_nothing(self, source, storage, org): + output = self._run("--source", str(source)) + assert "copied_from_disk=1" in output and "rows_renamed=1" in output + assert not storage.exists(self.NEW) + org.refresh_from_db() + assert org.logo.name == self.OLD + + def test_disk_file_lands_under_the_clean_name_and_row_follows(self, source, storage, org): + assert "copied_from_disk=1" in self._run("--source", str(source), "--execute") + assert self._read(storage, self.NEW) == b"logo-bytes" + org.refresh_from_db() + assert org.logo.name == self.NEW + assert "in_place=1" in self._run("--source", str(source), "--execute") + + def test_old_s3_key_is_copied_to_the_clean_name(self, storage, org): + storage.save(self.OLD, ContentFile(b"from-s3")) + assert "moved_in_s3=1" in self._run("--execute") + assert self._read(storage, self.NEW) == b"from-s3" + assert storage.exists(self.OLD) # nothing is deleted + org.refresh_from_db() + assert org.logo.name == self.NEW + + def test_counts_missing_and_orphaned_files(self, source, storage, org): + Organization.objects.create(name="Gone", slug="gone", logo="files/public/gone.png") + output = self._run("--source", str(source)) + assert "missing=1" in output + assert "orphaned=1" in output + + def test_a_missing_file_keeps_its_row_name(self, storage): + gone = Organization.objects.create(name="Gone", slug="gone", logo="files/public/gone.png") + self._run("--execute") + gone.refresh_from_db() + assert gone.logo.name == "files/public/gone.png" + + def test_a_failed_copy_exits_non_zero(self, source, storage, org): + with patch.object(storage, "save", side_effect=OSError("denied")): + with pytest.raises(CommandError): + self._run("--source", str(source), "--execute") + org.refresh_from_db() + assert org.logo.name == self.OLD diff --git a/tests/test_settings.py b/tests/test_settings.py index 6a641855..2159f0b0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -9,8 +9,22 @@ project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, project_root) +# The real settings require the S3 storage variables; give the suite harmless +# values so it boots without a .env. STORAGES is swapped for memory below. +os.environ.setdefault("STORAGE_BUCKET", "test-bucket") +os.environ.setdefault("STORAGE_REGION", "ap-south-1") +os.environ.setdefault("STORAGE_ENV", "test") + from DataSpace.settings import * +# Tests never reach S3: both media storages (and the default) live in memory. +STORAGES = { + **STORAGES, # noqa: F405 — imported via ``from DataSpace.settings import *`` + "default": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, + "public": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, + "private": {"BACKEND": "django.core.files.storage.InMemoryStorage"}, +} + # Use an in-memory SQLite database for testing DATABASES = { "default": { @@ -19,6 +33,18 @@ } } +# Data tables live in a second Postgres database. Tests that build them +# (marked ``data_db``) run only when TEST_DATA_DB_URL points at a Postgres +# server; pytest-django creates and drops its own database there. The router +# keeps every Django model out of it, as in production. +_TEST_DATA_DB_URL = os.environ.get("TEST_DATA_DB_URL") +if _TEST_DATA_DB_URL: + DATABASES["data_db"] = { + **env.db_url_config(_TEST_DATA_DB_URL), # noqa: F405 — from DataSpace.settings + "TEST": {"NAME": "test_dataspace_data_db"}, + } + DATABASE_ROUTERS = ["tests.data_index.routers.DataDbTestRouter"] + # Disable debug mode DEBUG = True diff --git a/tests/test_slug_collision.py b/tests/test_slug_collision.py index c94263bf..7cddb8d9 100644 --- a/tests/test_slug_collision.py +++ b/tests/test_slug_collision.py @@ -1,33 +1,32 @@ """Regression test for the same-second slug/title collision. -Root cause: auto-generated titles (e.g. "New dataset 17 Sep 2026 - 09:43:04") -have one-second resolution, and Dataset.slug / UseCase.title+slug / -Collaborative.title+slug are unique columns. Two creates landing in the same -second (routine under concurrent CI load - e.g. `-n 3`) previously raised an -IntegrityError/ValidationError that the create mutation surfaced as a bare -`success: False` with no usable error, which is what made -test_prv_006_org_create_dataset and test_prv_007_org_create_usecase in -CivicDataSpace-test fail under concurrent load (DataSpaceBackend #199). - -Model.save() now retries with a disambiguated title/slug on collision -(api/utils/django_utils.py:retry_on_slug_collision), so this same-second -create no longer fails. +Auto-generated titles (e.g. "New dataset 17 Sep 2026 - 09:43:04") have +one-second resolution, so two creates in the same second get the same title. +Titles and slugs used to be unique, so the second create failed under +concurrent load (DataSpaceBackend #199). + +Titles and slugs no longer need to be unique: each record gets its own +public_id, and links are found by that id. Two records may share a title. """ import pytest -from api.models import Collaborative, Dataset, UseCase +from api.models import Collaborative, Dataset, Resource, UseCase from authorization.models import User @pytest.mark.django_db def test_dataset_same_second_title_does_not_collide(): + # Dataset slugs are no longer unique; each dataset is found by its own + # public_id, so two datasets may share a title and slug. same_title = "New dataset 18 Sep 2026 - 09:43:04" first = Dataset.objects.create(title=same_title) second = Dataset.objects.create(title=same_title) - assert first.slug != second.slug - assert second.slug.startswith(first.slug) + assert first.slug == second.slug + assert first.public_id and second.public_id + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug @pytest.mark.django_db @@ -38,9 +37,9 @@ def test_usecase_same_second_title_does_not_collide(): first = UseCase.objects.create(title=same_title, user=user) second = UseCase.objects.create(title=same_title, user=user) - assert first.title == same_title - assert second.title != same_title - assert second.slug != first.slug + assert first.title == second.title == same_title + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug @pytest.mark.django_db @@ -51,6 +50,28 @@ def test_collaborative_same_second_title_does_not_collide(): first = Collaborative.objects.create(title=same_title, user=user) second = Collaborative.objects.create(title=same_title, user=user) - assert first.title == same_title - assert second.title != same_title - assert second.slug != first.slug + assert first.title == second.title == same_title + # The slug is also the subdomain, so it stays unique. + assert first.slug != second.slug + assert first.public_id != second.public_id + assert first.url_slug != second.url_slug + + +@pytest.mark.django_db +def test_resources_with_the_same_name_do_not_collide(): + # Resource slugs are no longer unique (#229): two uploads of the same file + # name used to race on the unique rule. + dataset = Dataset.objects.create(title="Resource collision") + first = Resource.objects.create(dataset=dataset, name="data.csv") + second = Resource.objects.create(dataset=dataset, name="data.csv") + + assert first.slug == second.slug == "datacsv" + assert first.pk != second.pk + + +@pytest.mark.django_db +def test_resource_with_a_preset_slug_keeps_it(): + dataset = Dataset.objects.create(title="Resource preset slug") + resource = Resource.objects.create(dataset=dataset, name="data.csv", slug="my-file") + + assert resource.slug == "my-file" diff --git a/tests/test_source_identifier_migration.py b/tests/test_source_identifier_migration.py new file mode 100644 index 00000000..1f9d6e94 --- /dev/null +++ b/tests/test_source_identifier_migration.py @@ -0,0 +1,78 @@ +"""Tests for the migration that adds the "Source identifier" dataset metadata item. + +CivicDataLab's scrapers look the item up by its exact label, on datasets, and +only while it is switched on, so these check that the migration leaves exactly +one such item behind, switched on, whatever state it finds. + +The test settings skip migrations, so the migration's function is called +directly against the current models. +""" + +import importlib + +import pytest +from django.apps import apps + +from api.models import Metadata + +migration = importlib.import_module("api.migrations.0054_add_source_identifier_metadata") + + +def source_identifier_items(): + return Metadata.objects.filter(label="Source identifier", model="DATASET") + + +@pytest.mark.django_db +def test_creates_the_item_switched_on_when_missing(): + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "dataspace-scrapers:source-identifier" + assert item.data_type == "STRING" + assert item.type == "OPTIONAL" + assert item.filterable is False + + +@pytest.mark.django_db +def test_switches_an_existing_item_back_on_without_adding_another(): + Metadata.objects.create( + label="Source identifier", + model="DATASET", + urn="admin-made:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + item = source_identifier_items().get() + assert item.enabled is True + assert item.urn == "admin-made:source-identifier" + + +@pytest.mark.django_db +def test_running_twice_leaves_one_item(): + migration.add_source_identifier(apps, None) + migration.add_source_identifier(apps, None) + + assert source_identifier_items().count() == 1 + + +@pytest.mark.django_db +def test_leaves_a_same_named_item_on_another_model_alone(): + resource_item = Metadata.objects.create( + label="Source identifier", + model="RESOURCE", + urn="resource:source-identifier", + data_type="STRING", + type="OPTIONAL", + enabled=False, + ) + + migration.add_source_identifier(apps, None) + + resource_item.refresh_from_db() + assert resource_item.enabled is False + assert source_identifier_items().get().urn == "dataspace-scrapers:source-identifier"