diff --git a/backend/app/api/v1/endpoint_modules/ogm.py b/backend/app/api/v1/endpoint_modules/ogm.py index c72b167..9b986d6 100644 --- a/backend/app/api/v1/endpoint_modules/ogm.py +++ b/backend/app/api/v1/endpoint_modules/ogm.py @@ -78,7 +78,17 @@ async def ogm_repo_dashboard(request: Request): or f"https://github.com/OpenGeoMetadata/{quote(name, safe='')}", "ogm_search_url": "/api/v1/search?" + urlencode({"ogm_repo": name}), "display_last_commit_at": _format_timestamp(repo.get("last_commit_at")), - "display_last_harvest_at": _format_timestamp(repo.get("last_crawl_completed_at")), + "display_last_successful_harvest_at": _format_timestamp( + repo.get("last_successful_harvest_at") + ), + "display_harvest_started_at": _format_timestamp(repo.get("last_crawl_started_at")), + "display_harvest_stage": { + "sync": "Fetching metadata", + "import": "Importing records", + "search_index": "Updating search index", + "thumbnail_refresh": "Refreshing thumbnails", + "dumps": "Saving harvest files", + }.get(repo.get("last_run_stage"), "Processing"), "source_status": "Nightly" if scheduled else "Not scheduled", "indexed_record_count": count, } @@ -90,6 +100,7 @@ async def ogm_repo_dashboard(request: Request): } if templates is None: + no_success = "No successful harvest recorded" rows = "".join( ( "" @@ -98,7 +109,16 @@ async def ogm_repo_dashboard(request: Request): f"({escape(repo['source_status'])}) " f'Search API' f"{escape(str(repo.get('display_last_commit_at') or '-'))}" - f"{escape(str(repo.get('display_last_harvest_at') or '-'))}" + "" + f"{escape(repo.get('display_last_successful_harvest_at') or no_success)}" + "" + f"{escape(str(repo.get('last_crawl_status') or 'Unknown'))}" + + ( + f" — {escape(repo['display_harvest_stage'])}" + if repo.get("last_crawl_status") == "running" + else "" + ) + + f"; Started {escape(repo.get('display_harvest_started_at') or 'Unknown')}" f"{repo['indexed_record_count']}" "" ) @@ -112,7 +132,8 @@ async def ogm_repo_dashboard(request: Request): "

OpenGeoMetadata Repository Dashboard

" "

Templates are unavailable, showing a minimal fallback view.

" "" - "" + "" + "" "" f"{rows}
RepositoryLast commitLast harvestRepositoryLast commitLast successful harvestLatest harvestIndexed records
" ) diff --git a/backend/app/services/ogm_harvest/repository.py b/backend/app/services/ogm_harvest/repository.py index ea17fe0..ae9b039 100644 --- a/backend/app/services/ogm_harvest/repository.py +++ b/backend/app/services/ogm_harvest/repository.py @@ -90,6 +90,15 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: .group_by(ogm_harvest_runs.c.ogm_repo_name) .subquery() ) + last_successful_runs = ( + select( + ogm_harvest_runs.c.ogm_repo_name, + func.max(ogm_harvest_runs.c.ogm_completed_at).label("last_successful_harvest_at"), + ) + .where(ogm_harvest_runs.c.ogm_status == "success") + .group_by(ogm_harvest_runs.c.ogm_repo_name) + .subquery("last_successful_runs") + ) harvested_counts = ( select( ogm_resource_state.c.ogm_repo_name, @@ -169,6 +178,7 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: ogm_repos.c.ogm_last_harvest_status, ogm_repos.c.ogm_last_commit_sha, ogm_repos.c.ogm_tags, + last_successful_runs.c.last_successful_harvest_at, ogm_harvest_runs.c.ogm_id.label("last_run_id"), ogm_harvest_runs.c.ogm_started_at.label("last_run_started_at"), ogm_harvest_runs.c.ogm_completed_at.label("last_run_completed_at"), @@ -192,6 +202,10 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]: other_active_counts, ogm_repos.c.ogm_repo_name == other_active_counts.c.ogm_repo_name, ) + .outerjoin( + last_successful_runs, + last_successful_runs.c.ogm_repo_name == ogm_repos.c.ogm_repo_name, + ) .outerjoin(latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name) .outerjoin( ogm_harvest_runs, ogm_harvest_runs.c.ogm_id == latest_run_ids.c.latest_ogm_id @@ -225,8 +239,11 @@ def _to_int(value: Any) -> int: latest_started_at = item.get("last_run_started_at") or item.get( "ogm_last_harvest_started_at" ) - latest_completed_at = item.get("last_run_completed_at") or item.get( - "ogm_last_harvest_completed_at" + # A running run has no completion time; never borrow an older run's timestamp. + latest_completed_at = ( + item.get("last_run_completed_at") + if item.get("last_run_id") is not None + else item.get("ogm_last_harvest_completed_at") ) repo_full_name = tags.get("ogm_repo_full_name") or item.get("ogm_repo_name") if repo_full_name and "/" not in repo_full_name: @@ -256,6 +273,8 @@ def _to_int(value: Any) -> int: "last_crawl_completed_at": latest_completed_at, "last_crawl_status": latest_status, "last_run_id": item.get("last_run_id"), + "last_run_stage": stats.get("stage") if latest_status == "running" else None, + "last_successful_harvest_at": item.get("last_successful_harvest_at"), "harvested_success_count": _to_int(stats.get("imported")), "harvested_failure_count": _to_int(stats.get("errors")), "harvested_record_count": _to_int(item.get("harvested_record_count")), diff --git a/backend/templates/ogm_repo_dashboard.html b/backend/templates/ogm_repo_dashboard.html index 8085e19..532712d 100644 --- a/backend/templates/ogm_repo_dashboard.html +++ b/backend/templates/ogm_repo_dashboard.html @@ -72,8 +72,8 @@

Repositories

Repository Last Commit - Last Harvest - Last Harvest Result + Last Successful Harvest + Latest Harvest Indexed Records @@ -92,10 +92,10 @@

Repositories

{{ repo.display_last_commit_at or "Never observed" }} - {% if repo.display_last_harvest_at %} -
{{ repo.display_last_harvest_at }}
+ {% if repo.display_last_successful_harvest_at %} +
{{ repo.display_last_successful_harvest_at }}
{% else %} - Never harvested + No successful harvest recorded {% endif %} @@ -104,11 +104,15 @@

Repositories

Success {% elif crawl_status == "running" %} Running +

{{ repo.display_harvest_stage }}

{% elif crawl_status == "failed" %} Failed {% else %} {{ repo.last_crawl_status or "Unknown" }} {% endif %} + {% if repo.display_harvest_started_at %} +

Started {{ repo.display_harvest_started_at }}

+ {% endif %} {% if repo.harvested_failure_count %}

{{ "{:,}".format(repo.harvested_failure_count) }} recent import errors

{% endif %} diff --git a/backend/tests/api/v1/test_ogm_public_endpoints.py b/backend/tests/api/v1/test_ogm_public_endpoints.py index 80ad0a6..02ed6d2 100644 --- a/backend/tests/api/v1/test_ogm_public_endpoints.py +++ b/backend/tests/api/v1/test_ogm_public_endpoints.py @@ -1,5 +1,6 @@ from unittest.mock import AsyncMock, patch +import pytest from fastapi import FastAPI from fastapi.testclient import TestClient @@ -221,3 +222,63 @@ def test_dashboard_empty_index(): response = client.get("/api/v1/ogm/repos/dashboard") assert response.status_code == 200 assert "No repositories currently have indexed records" in response.text + + +@pytest.mark.parametrize( + "stage,label", + [ + ("sync", "Fetching metadata"), + ("import", "Importing records"), + ("search_index", "Updating search index"), + ("thumbnail_refresh", "Refreshing thumbnails"), + ("dumps", "Saving harvest files"), + (None, "Processing"), + ("future_stage", "Processing"), + ], +) +def test_dashboard_separates_running_progress_from_last_success(stage, label): + repo = { + "ogm_repo_name": "gov.usgs.htmc", + "last_crawl_status": "running", + "last_crawl_started_at": "2026-10-04T13:20:00", + "last_successful_harvest_at": "2026-10-03T13:10:00", + "last_run_stage": stage, + } + with ( + patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=[repo])), + patch.object( + ogm, + "get_index_status", + AsyncMock( + return_value={"record_count": 177789, "repo_counts": {"gov.usgs.htmc": 177789}} + ), + ), + ): + html = client.get("/api/v1/ogm/repos/dashboard").text + assert "Last Successful Harvest" in html + assert "2026-10-03 13:10 UTC" in html + assert "Started 2026-10-04 13:20 UTC" in html + assert label in html + assert "Running" in html + + +@pytest.mark.parametrize("status", ["failed", "success", "running"]) +def test_dashboard_does_not_invent_success_from_latest_completion(status): + repo = { + "ogm_repo_name": "example", + "last_crawl_status": status, + "last_crawl_completed_at": "2026-10-04T14:00:00", + "last_run_stage": "search_index", + } + with ( + patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=[repo])), + patch.object( + ogm, + "get_index_status", + AsyncMock(return_value={"record_count": 1, "repo_counts": {"example": 1}}), + ), + ): + html = client.get("/api/v1/ogm/repos/dashboard").text + assert "No successful harvest recorded" in html + assert "2026-10-04 14:00 UTC" not in html + assert ("Updating search index" in html) == (status == "running") diff --git a/backend/tests/services/test_ogm_harvest_repository.py b/backend/tests/services/test_ogm_harvest_repository.py index 1fc757a..71e6956 100644 --- a/backend/tests/services/test_ogm_harvest_repository.py +++ b/backend/tests/services/test_ogm_harvest_repository.py @@ -120,6 +120,33 @@ async def fetch_all(query): assert summaries[0]["last_crawl_status"] == "success" assert summaries[1]["ogm_enabled"] is False assert summaries[2]["last_run_id"] is None + # A newer running or failed run must not replace the last successful completion. + await connection.execute(""" + UPDATE ogm_harvest_runs SET ogm_completed_at='2026-10-03 13:10:00' + WHERE ogm_id=2; + UPDATE ogm_repos SET ogm_last_harvest_completed_at='2026-10-03 13:10:00' + WHERE ogm_repo_name='alpha'; + INSERT INTO ogm_harvest_runs + (ogm_id,ogm_repo_name,ogm_trigger,ogm_status,ogm_started_at,ogm_stats_json) + VALUES (3,'alpha','nightly','running','2026-10-04 13:20:00', + '{"stage":"search_index"}'); + """) + current = (await OGMHarvestRepository().list_public_repo_summaries())[0] + assert current["last_crawl_completed_at"] is None + assert str(current["last_crawl_started_at"]) == "2026-10-04 13:20:00" + assert str(current["last_successful_harvest_at"]) == "2026-10-03 13:10:00" + assert current["last_run_stage"] == "search_index" + await connection.execute(""" + UPDATE ogm_harvest_runs SET ogm_status='failed', + ogm_completed_at='2026-10-04 14:00:00' WHERE ogm_id=3; + """) + current = (await OGMHarvestRepository().list_public_repo_summaries())[0] + assert str(current["last_successful_harvest_at"]) == "2026-10-03 13:10:00" + assert str(current["last_crawl_completed_at"]) == "2026-10-04 14:00:00" + assert current["last_run_stage"] is None + assert (await OGMHarvestRepository().list_public_repo_summaries())[1][ + "last_successful_harvest_at" + ] is None # A second dataset covers migrations, overlapping active sources, missing # records, and published records without repository attribution. await connection.execute("""