feat: Milestone 2 — Structured Run Manifest System (#64)

.gitignore

@@ -30,6 +30,11 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+# Explicitly un-ignore the manifest module folder (MANIFEST above is for Python packaging artifacts)
+!databusclient/manifest/
+!databusclient/manifest/**
+databusclient/manifest/__pycache__/
+*.py[cod]
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

README.md

@@ -675,4 +675,30 @@ Or to ensure compatibility with the `pyproject.toml` configured dependencies, ru
 
 ```bash
 poetry run pytest tests/
-```
+```
+
+## Manifest
+
+All three commands support an optional `--manifest` flag that writes a structured JSON-LD record of the operation to disk:
+
+```bash
+databusclient download https://databus.dbpedia.org/dbpedia/generic/labels/2023.12.01 \
+  --manifest ./manifests/labels-download.jsonld
+
+databusclient deploy --version-id https://databus.dbpedia.org/myaccount/mygroup/mydata/1.0 \
+  --title "My Dataset" --abstract "..." --description "..." \
+  --license https://creativecommons.org/licenses/by-sa/3.0/ \
+  --apikey YOUR_KEY --manifest ./manifests/deploy-run.jsonld \
+  myfile.nt
+
+databusclient delete https://databus.dbpedia.org/myaccount/mygroup/mydata/1.0 \
+  --databus-key YOUR_KEY --manifest ./manifests/delete-run.jsonld
+```
+
+The manifest records input parameters, per-file URLs, checksums, byte sizes, timestamps, and success/failure status for each file. It uses the DataID vocabulary and is versioned via `dbus:schemaVersion`.
+
+- If the target path already exists, the manifest is written to an auto-suffixed path (e.g. `run_1.jsonld`) with a warning.
+- Sensitive fields (API keys, vault tokens) are never written.
+- If manifest writing fails, a warning is printed and the exit code reflects the actual operation result.
+
+See `examples/reproducible-download.md` for a full walkthrough.

databusclient/api/delete.py

@@ -116,6 +116,7 @@ def _delete_resource(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a single Databus resource (version, artifact, group).
 
@@ -144,6 +145,8 @@ def _delete_resource(
 
     if dry_run:
         print(f"[DRY RUN] Would delete: {databusURI}")
+        if manifest_context is not None:
+            manifest_context.record_file(url=databusURI, status="dry_run")
         return
 
     if queue is not None:
@@ -156,6 +159,8 @@ def _delete_resource(
 
     if response.status_code in (200, 204):
         print(f"Successfully deleted: {databusURI}")
+        if manifest_context is not None:
+            manifest_context.record_file(url=databusURI, status="success")
     else:
         raise Exception(
             f"Failed to delete {databusURI}: {response.status_code} - {response.text}"
@@ -168,6 +173,7 @@ def _delete_list(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a list of Databus resources.
 
@@ -180,7 +186,7 @@ def _delete_list(
     """
     for databusURI in databusURIs:
         _delete_resource(
-            databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+            databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
         )
 
 
@@ -190,6 +196,7 @@ def _delete_artifact(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete an artifact and all its versions.
 
@@ -223,11 +230,11 @@ def _delete_artifact(
         else:
             # Delete all versions
             _delete_list(
-                version_uris, databus_key, dry_run=dry_run, force=force, queue=queue
+                version_uris, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
 
     # Finally, delete the artifact itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue)
+    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
 
 
 def _delete_group(
@@ -236,6 +243,7 @@ def _delete_group(
     dry_run: bool = False,
     force: bool = False,
     queue: DeleteQueue = None,
+    manifest_context=None,
 ):
     """Delete a group and all its artifacts and versions.
 
@@ -266,14 +274,14 @@ def _delete_group(
     # Delete all artifacts (which deletes their versions)
     for artifact_uri in artifact_uris:
         _delete_artifact(
-            artifact_uri, databus_key, dry_run=dry_run, force=force, queue=queue
+            artifact_uri, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
         )
 
     # Finally, delete the group itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue)
+    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
 
 
-def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool):
+def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool, manifest_context=None):
     """Delete a dataset from the databus.
 
     Delete a group, artifact, or version identified by the given databus URI.
@@ -296,24 +304,24 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool)
         if group == "collections" and artifact is not None:
             print(f"Deleting collection: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif file is not None:
             print(f"Deleting file is not supported via API: {databusURI}")
         elif version is not None:
             print(f"Deleting version: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif artifact is not None:
             print(f"Deleting artifact and all its versions: {databusURI}")
             _delete_artifact(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         elif group is not None and group != "collections":
             print(f"Deleting group and all its artifacts and versions: {databusURI}")
             _delete_group(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue
+                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
             )
         else:
             print(f"Deleting {databusURI} is not supported.")

databusclient/api/download.py

@@ -12,6 +12,7 @@
 import requests
 from SPARQLWrapper import JSON, SPARQLWrapper
 from tqdm import tqdm
+from datetime import datetime, timezone
 
 from databusclient.api.utils import (
     fetch_databus_jsonld,
@@ -327,6 +328,7 @@ def _download_file(
     base_uri=None,
     validate_checksum: bool = False,
     expected_checksum: str | None = None,
+    manifest_context=None,
 ) -> None:
     """Download a file from the internet with a progress bar using tqdm.
 
@@ -495,32 +497,43 @@ def _download_file(
         raise IOError("Downloaded size does not match Content-Length header")
 
     # --- 6. Validate checksum on original downloaded file (BEFORE conversion) ---
+    actual_checksum = None
     if validate_checksum:
-        # reuse compute_sha256_and_length from webdav extension
         try:
-            actual, _ = compute_sha256_and_length(filename)
+            actual_checksum, _ = compute_sha256_and_length(filename)
         except (OSError, IOError) as e:
             print(f"WARNING: error computing checksum for {filename}: {e}")
-            actual = None
+            actual_checksum = None
 
         if expected_checksum is None:
             print(
                 f"WARNING: no expected checksum available for {filename}; skipping validation"
             )
-        elif actual is None:
+        elif actual_checksum is None:
             print(
                 f"WARNING: could not compute checksum for {filename}; skipping validation"
             )
         else:
-            if actual.lower() != expected_checksum.lower():
+            if actual_checksum.lower() != expected_checksum.lower():
                 try:
-                    os.remove(filename)  # delete corrupted file
+                    os.remove(filename)
                 except OSError:
                     pass
                 raise IOError(
-                    f"Checksum mismatch for {filename}: expected {expected_checksum}, got {actual}"
+                    f"Checksum mismatch for {filename}: expected {expected_checksum}, got {actual_checksum}"
                 )
 
+    # Record file to manifest after all verification passes.
+    # Use actual computed checksum if available, otherwise fall back to expected.
+    if manifest_context is not None:
+        manifest_context.record_file(
+            url=url,
+            status="success",
+            sha256=actual_checksum or expected_checksum,
+            size_bytes=total_size_in_bytes if total_size_in_bytes else None,
+            downloaded_at=datetime.now(timezone.utc).isoformat(),
+        )
+
     # --- 7. Unified compression/format conversion pass ---
     source_compression = _detect_compression_format(file)
     should_convert_compression, source_fmt = _should_convert_compression(
@@ -713,6 +726,7 @@ def _download_files(
     convert_format: str = None,
     graph_name: str = None,
     base_uri: str = None,
+    manifest_context=None,
     validate_checksum: bool = False,
     checksums: dict | None = None,
 ) -> None:
@@ -749,6 +763,7 @@ def _download_files(
             base_uri=base_uri,
             validate_checksum=validate_checksum,
             expected_checksum=expected,
+            manifest_context=manifest_context,
         )
 
 def _get_sparql_query_of_collection(uri: str, databus_key: str | None = None) -> str:
@@ -896,6 +911,7 @@ def _download_collection(
     convert_format: str = None,
     graph_name: str = None,
     base_uri: str = None,
+    manifest_context=None,
     validate_checksum: bool = False,
 ) -> None:
     """Download all files in a databus collection.
@@ -935,6 +951,7 @@ def _download_collection(
         convert_format=convert_format,
         graph_name=graph_name,
         base_uri=base_uri,
+        manifest_context=manifest_context,
         validate_checksum=validate_checksum,
         checksums=checksums if checksums else None,
     )
@@ -951,6 +968,7 @@ def _download_version(
     convert_format: str = None,
     graph_name: str = None,
     base_uri: str = None,
+    manifest_context=None,
     validate_checksum: bool = False,
 ) -> None:
     """Download all files in a databus artifact version.
@@ -988,6 +1006,7 @@ def _download_version(
         convert_format=convert_format,
         graph_name=graph_name,
         base_uri=base_uri,
+        manifest_context=manifest_context,
         validate_checksum=validate_checksum,
         checksums=checksums,
     )
@@ -1005,6 +1024,7 @@ def _download_artifact(
     convert_format: str = None,
     graph_name: str = None,
     base_uri: str = None,
+    manifest_context=None,
     validate_checksum: bool = False,
 ) -> None:
     """Download files in a databus artifact.
@@ -1049,6 +1069,7 @@ def _download_artifact(
             convert_format=convert_format,
             graph_name=graph_name,
             base_uri=base_uri,
+            manifest_context=manifest_context,
             validate_checksum=validate_checksum,
             checksums=checksums,
         )
@@ -1127,6 +1148,7 @@ def _download_group(
     convert_format: str = None,
     graph_name: str = None,
     base_uri: str = None,
+    manifest_context=None,
     validate_checksum: bool = False,
 ) -> None:
     """Download files in a databus group.
@@ -1161,6 +1183,7 @@ def _download_group(
             convert_format=convert_format,
             graph_name=graph_name,
             base_uri=base_uri,
+            manifest_context=manifest_context,
             validate_checksum=validate_checksum,
         )
 
@@ -1213,6 +1236,7 @@ def download(
     graph_name=None,
     base_uri=None,
     validate_checksum: bool = False,
+    manifest_context=None,
 ) -> None:
     """Download datasets from databus.
 
@@ -1262,6 +1286,7 @@ def download(
                     convert_format,
                     graph_name=graph_name,
                     base_uri=base_uri,
+                    manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                 )
             elif file is not None:
@@ -1285,6 +1310,7 @@ def download(
                     convert_format=convert_format,
                     graph_name=graph_name,
                     base_uri=base_uri,
+                    manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                     expected_checksum=expected,
                 )
@@ -1301,6 +1327,7 @@ def download(
                     convert_format=convert_format,
                     graph_name=graph_name,
                     base_uri=base_uri,
+                    manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                 )
             elif artifact is not None:
@@ -1319,6 +1346,7 @@ def download(
                     convert_format=convert_format,
                     graph_name=graph_name,
                     base_uri=base_uri,
+                    manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                 )
             elif group is not None and group != "collections":
@@ -1337,6 +1365,7 @@ def download(
                     convert_format=convert_format,
                     graph_name=graph_name,
                     base_uri=base_uri,
+                    manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                 )
             elif account is not None:
@@ -1377,6 +1406,7 @@ def download(
                 convert_format=convert_format,
                 graph_name=graph_name,
                 base_uri=base_uri,
+                manifest_context=manifest_context,
                 validate_checksum=validate_checksum,
                 checksums=checksums if checksums else None,
             )