Show current harvest progress separately from last success

backend/app/api/v1/endpoint_modules/ogm.py

@@ -78,7 +78,17 @@ async def ogm_repo_dashboard(request: Request):
                 or f"https://github.com/OpenGeoMetadata/{quote(name, safe='')}",
                 "ogm_search_url": "/api/v1/search?" + urlencode({"ogm_repo": name}),
                 "display_last_commit_at": _format_timestamp(repo.get("last_commit_at")),
-                "display_last_harvest_at": _format_timestamp(repo.get("last_crawl_completed_at")),
+                "display_last_successful_harvest_at": _format_timestamp(
+                    repo.get("last_successful_harvest_at")
+                ),
+                "display_harvest_started_at": _format_timestamp(repo.get("last_crawl_started_at")),
+                "display_harvest_stage": {
+                    "sync": "Fetching metadata",
+                    "import": "Importing records",
+                    "search_index": "Updating search index",
+                    "thumbnail_refresh": "Refreshing thumbnails",
+                    "dumps": "Saving harvest files",
+                }.get(repo.get("last_run_stage"), "Processing"),
                 "source_status": "Nightly" if scheduled else "Not scheduled",
                 "indexed_record_count": count,
             }
@@ -90,6 +100,7 @@ async def ogm_repo_dashboard(request: Request):
     }
 
     if templates is None:
+        no_success = "No successful harvest recorded"
         rows = "".join(
             (
                 "<tr>"
@@ -98,7 +109,16 @@ async def ogm_repo_dashboard(request: Request):
                 f"({escape(repo['source_status'])}) "
                 f'<a href="{escape(repo["ogm_search_url"], quote=True)}">Search API</a></td>'
                 f"<td>{escape(str(repo.get('display_last_commit_at') or '-'))}</td>"
-                f"<td>{escape(str(repo.get('display_last_harvest_at') or '-'))}</td>"
+                "<td>"
+                f"{escape(repo.get('display_last_successful_harvest_at') or no_success)}"
+                "</td>"
+                f"<td>{escape(str(repo.get('last_crawl_status') or 'Unknown'))}"
+                + (
+                    f" — {escape(repo['display_harvest_stage'])}"
+                    if repo.get("last_crawl_status") == "running"
+                    else ""
+                )
+                + f"; Started {escape(repo.get('display_harvest_started_at') or 'Unknown')}</td>"
                 f"<td>{repo['indexed_record_count']}</td>"
                 "</tr>"
             )
@@ -112,7 +132,8 @@ async def ogm_repo_dashboard(request: Request):
                 "<body><h1>OpenGeoMetadata Repository Dashboard</h1>"
                 "<p>Templates are unavailable, showing a minimal fallback view.</p>"
                 "<table><thead><tr>"
-                "<th>Repository</th><th>Last commit</th><th>Last harvest</th>"
+                "<th>Repository</th><th>Last commit</th><th>Last successful harvest</th>"
+                "<th>Latest harvest</th>"
                 "<th>Indexed records</th>"
                 f"</tr></thead><tbody>{rows}</tbody></table></body></html>"
             )

backend/app/services/ogm_harvest/repository.py

@@ -90,6 +90,15 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
             .group_by(ogm_harvest_runs.c.ogm_repo_name)
             .subquery()
         )
+        last_successful_runs = (
+            select(
+                ogm_harvest_runs.c.ogm_repo_name,
+                func.max(ogm_harvest_runs.c.ogm_completed_at).label("last_successful_harvest_at"),
+            )
+            .where(ogm_harvest_runs.c.ogm_status == "success")
+            .group_by(ogm_harvest_runs.c.ogm_repo_name)
+            .subquery("last_successful_runs")
+        )
         harvested_counts = (
             select(
                 ogm_resource_state.c.ogm_repo_name,
@@ -169,6 +178,7 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
                 ogm_repos.c.ogm_last_harvest_status,
                 ogm_repos.c.ogm_last_commit_sha,
                 ogm_repos.c.ogm_tags,
+                last_successful_runs.c.last_successful_harvest_at,
                 ogm_harvest_runs.c.ogm_id.label("last_run_id"),
                 ogm_harvest_runs.c.ogm_started_at.label("last_run_started_at"),
                 ogm_harvest_runs.c.ogm_completed_at.label("last_run_completed_at"),
@@ -192,6 +202,10 @@ async def list_public_repo_summaries(self) -> List[Dict[str, Any]]:
                     other_active_counts,
                     ogm_repos.c.ogm_repo_name == other_active_counts.c.ogm_repo_name,
                 )
+                .outerjoin(
+                    last_successful_runs,
+                    last_successful_runs.c.ogm_repo_name == ogm_repos.c.ogm_repo_name,
+                )
                 .outerjoin(latest_run_ids, ogm_repos.c.ogm_repo_name == latest_run_ids.c.repo_name)
                 .outerjoin(
                     ogm_harvest_runs, ogm_harvest_runs.c.ogm_id == latest_run_ids.c.latest_ogm_id
@@ -225,8 +239,11 @@ def _to_int(value: Any) -> int:
             latest_started_at = item.get("last_run_started_at") or item.get(
                 "ogm_last_harvest_started_at"
             )
-            latest_completed_at = item.get("last_run_completed_at") or item.get(
-                "ogm_last_harvest_completed_at"
+            # A running run has no completion time; never borrow an older run's timestamp.
+            latest_completed_at = (
+                item.get("last_run_completed_at")
+                if item.get("last_run_id") is not None
+                else item.get("ogm_last_harvest_completed_at")
             )
             repo_full_name = tags.get("ogm_repo_full_name") or item.get("ogm_repo_name")
             if repo_full_name and "/" not in repo_full_name:
@@ -256,6 +273,8 @@ def _to_int(value: Any) -> int:
                     "last_crawl_completed_at": latest_completed_at,
                     "last_crawl_status": latest_status,
                     "last_run_id": item.get("last_run_id"),
+                    "last_run_stage": stats.get("stage") if latest_status == "running" else None,
+                    "last_successful_harvest_at": item.get("last_successful_harvest_at"),
                     "harvested_success_count": _to_int(stats.get("imported")),
                     "harvested_failure_count": _to_int(stats.get("errors")),
                     "harvested_record_count": _to_int(item.get("harvested_record_count")),

backend/templates/ogm_repo_dashboard.html

@@ -72,8 +72,8 @@ <h3>Repositories</h3>
             <tr>
               <th scope="col">Repository</th>
               <th scope="col">Last Commit</th>
-              <th scope="col">Last Harvest</th>
-              <th scope="col">Last Harvest Result</th>
+              <th scope="col">Last Successful Harvest</th>
+              <th scope="col">Latest Harvest</th>
               <th scope="col">Indexed Records</th>
             </tr>
           </thead>
@@ -92,10 +92,10 @@ <h3>Repositories</h3>
                 {{ repo.display_last_commit_at or "Never observed" }}
               </td>
               <td>
-                {% if repo.display_last_harvest_at %}
-                <div>{{ repo.display_last_harvest_at }}</div>
+                {% if repo.display_last_successful_harvest_at %}
+                <div>{{ repo.display_last_successful_harvest_at }}</div>
                 {% else %}
-                Never harvested
+                No successful harvest recorded
                 {% endif %}
               </td>
               <td>
@@ -104,11 +104,15 @@ <h3>Repositories</h3>
                 <span class="status-pill status-ok">Success</span>
                 {% elif crawl_status == "running" %}
                 <span class="status-pill status-run">Running</span>
+                <p>{{ repo.display_harvest_stage }}</p>
                 {% elif crawl_status == "failed" %}
                 <span class="status-pill status-bad">Failed</span>
                 {% else %}
                 <span class="status-pill status-muted">{{ repo.last_crawl_status or "Unknown" }}</span>
                 {% endif %}
+                {% if repo.display_harvest_started_at %}
+                <p class="repo-subtitle">Started {{ repo.display_harvest_started_at }}</p>
+                {% endif %}
                 {% if repo.harvested_failure_count %}
                 <p class="repo-subtitle">{{ "{:,}".format(repo.harvested_failure_count) }} recent import errors</p>
                 {% endif %}

backend/tests/api/v1/test_ogm_public_endpoints.py

@@ -1,5 +1,6 @@
 from unittest.mock import AsyncMock, patch
 
+import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
@@ -221,3 +222,63 @@ def test_dashboard_empty_index():
         response = client.get("/api/v1/ogm/repos/dashboard")
     assert response.status_code == 200
     assert "No repositories currently have indexed records" in response.text
+
+
+@pytest.mark.parametrize(
+    "stage,label",
+    [
+        ("sync", "Fetching metadata"),
+        ("import", "Importing records"),
+        ("search_index", "Updating search index"),
+        ("thumbnail_refresh", "Refreshing thumbnails"),
+        ("dumps", "Saving harvest files"),
+        (None, "Processing"),
+        ("future_stage", "Processing"),
+    ],
+)
+def test_dashboard_separates_running_progress_from_last_success(stage, label):
+    repo = {
+        "ogm_repo_name": "gov.usgs.htmc",
+        "last_crawl_status": "running",
+        "last_crawl_started_at": "2026-10-04T13:20:00",
+        "last_successful_harvest_at": "2026-10-03T13:10:00",
+        "last_run_stage": stage,
+    }
+    with (
+        patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=[repo])),
+        patch.object(
+            ogm,
+            "get_index_status",
+            AsyncMock(
+                return_value={"record_count": 177789, "repo_counts": {"gov.usgs.htmc": 177789}}
+            ),
+        ),
+    ):
+        html = client.get("/api/v1/ogm/repos/dashboard").text
+    assert "Last Successful Harvest" in html
+    assert "2026-10-03 13:10 UTC" in html
+    assert "Started 2026-10-04 13:20 UTC" in html
+    assert label in html
+    assert "Running" in html
+
+
+@pytest.mark.parametrize("status", ["failed", "success", "running"])
+def test_dashboard_does_not_invent_success_from_latest_completion(status):
+    repo = {
+        "ogm_repo_name": "example",
+        "last_crawl_status": status,
+        "last_crawl_completed_at": "2026-10-04T14:00:00",
+        "last_run_stage": "search_index",
+    }
+    with (
+        patch.object(ogm.ogm_repo, "list_public_repo_summaries", AsyncMock(return_value=[repo])),
+        patch.object(
+            ogm,
+            "get_index_status",
+            AsyncMock(return_value={"record_count": 1, "repo_counts": {"example": 1}}),
+        ),
+    ):
+        html = client.get("/api/v1/ogm/repos/dashboard").text
+    assert "No successful harvest recorded" in html
+    assert "2026-10-04 14:00 UTC" not in html
+    assert ("Updating search index" in html) == (status == "running")

backend/tests/services/test_ogm_harvest_repository.py

@@ -120,6 +120,33 @@ async def fetch_all(query):
         assert summaries[0]["last_crawl_status"] == "success"
         assert summaries[1]["ogm_enabled"] is False
         assert summaries[2]["last_run_id"] is None
+        # A newer running or failed run must not replace the last successful completion.
+        await connection.execute("""
+            UPDATE ogm_harvest_runs SET ogm_completed_at='2026-10-03 13:10:00'
+                WHERE ogm_id=2;
+            UPDATE ogm_repos SET ogm_last_harvest_completed_at='2026-10-03 13:10:00'
+                WHERE ogm_repo_name='alpha';
+            INSERT INTO ogm_harvest_runs
+                (ogm_id,ogm_repo_name,ogm_trigger,ogm_status,ogm_started_at,ogm_stats_json)
+            VALUES (3,'alpha','nightly','running','2026-10-04 13:20:00',
+                    '{"stage":"search_index"}');
+        """)
+        current = (await OGMHarvestRepository().list_public_repo_summaries())[0]
+        assert current["last_crawl_completed_at"] is None
+        assert str(current["last_crawl_started_at"]) == "2026-10-04 13:20:00"
+        assert str(current["last_successful_harvest_at"]) == "2026-10-03 13:10:00"
+        assert current["last_run_stage"] == "search_index"
+        await connection.execute("""
+            UPDATE ogm_harvest_runs SET ogm_status='failed',
+                ogm_completed_at='2026-10-04 14:00:00' WHERE ogm_id=3;
+        """)
+        current = (await OGMHarvestRepository().list_public_repo_summaries())[0]
+        assert str(current["last_successful_harvest_at"]) == "2026-10-03 13:10:00"
+        assert str(current["last_crawl_completed_at"]) == "2026-10-04 14:00:00"
+        assert current["last_run_stage"] is None
+        assert (await OGMHarvestRepository().list_public_repo_summaries())[1][
+            "last_successful_harvest_at"
+        ] is None
         # A second dataset covers migrations, overlapping active sources, missing
         # records, and published records without repository attribution.
         await connection.execute("""