Feat/argocd

.github/workflows/django.yml

@@ -5,6 +5,7 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
 
 jobs:
   lint:
@@ -83,11 +84,50 @@ jobs:
       - name: Run tests (pytest)
         run: uv run pytest -m "not slow"
 
+  manifests:
+    name: Kubernetes manifests
+    runs-on: ubuntu-latest
+    env:
+      KUSTOMIZE_VERSION: "5.8.1"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install pinned kustomize
+        run: |
+          curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \
+            | tar xz -C /usr/local/bin kustomize
+          kustomize version
+
+      - name: Render base
+        run: kustomize build kubernetes/base > /tmp/base.yaml
+
+      - name: Render ArgoCD test overlay
+        run: kustomize build kubernetes/overlays/minikube-argocd > /tmp/minikube-argocd.yaml
+
+      # overlays/home joins once its sealed/ files exist (requires the home
+      # cluster's sealed-secrets controller key).
+
+      - name: Assert sync-wave present on SealedSecrets
+        run: |
+          python3 - <<'EOF'
+          import sys, yaml
+          docs = list(yaml.safe_load_all(open("/tmp/minikube-argocd.yaml")))
+          sealed = [d for d in docs if d and d.get("kind") == "SealedSecret"]
+          assert sealed, "no SealedSecrets rendered"
+          for d in sealed:
+              wave = d["metadata"].get("annotations", {}).get("argocd.argoproj.io/sync-wave")
+              assert wave == "-3", f"{d['metadata']['name']}: sync-wave={wave!r}, expected '-3'"
+          print(f"OK: {len(sealed)} SealedSecrets at wave -3")
+          EOF
+
   publish:
     name: Publish app image to GHCR
     runs-on: ubuntu-latest
-    needs: [test, lint]
-    if: github.event_name == 'push'
+    needs: [test, lint, manifests]
+    # workflow_dispatch allows publishing a sha- tag from a feature branch
+    # (e.g. to test the branch on minikube before merge).
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
       contents: read
       packages: write
@@ -105,12 +145,86 @@ jobs:
       - name: Set lowercase owner
         run: echo "OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
 
+      - name: Set image tags
+        # The prod tag moves ONLY on main: Watchtower deploys whatever prod
+        # points at, so a branch dispatch must never touch it.
+        run: |
+          TAGS="ghcr.io/${OWNER}/ahc-app:sha-${{ github.sha }}"
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            TAGS="${TAGS},ghcr.io/${OWNER}/ahc-app:prod"
+          fi
+          echo "TAGS=${TAGS}" >> $GITHUB_ENV
+
       - name: Build and push ahc-app
         uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/Dockerfile-app
           push: true
-          tags: |
-            ghcr.io/${{ env.OWNER }}/ahc-app:prod
-            ghcr.io/${{ env.OWNER }}/ahc-app:sha-${{ github.sha }}
+          tags: ${{ env.TAGS }}
+
+  bump:
+    name: Bump image tag in home overlay
+    runs-on: ubuntu-latest
+    needs: publish
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    permissions:
+      contents: write
+    concurrency:
+      group: ahc-home-image-bump
+      cancel-in-progress: false
+    env:
+      KUSTOMIZE_VERSION: "5.8.1"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Install pinned kustomize
+        run: |
+          curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \
+            | tar xz -C /usr/local/bin kustomize
+          kustomize version
+
+      - name: Set lowercase owner
+        run: echo "OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
+
+      - name: Set image tag in home overlay
+        run: |
+          cd kubernetes/overlays/home
+          kustomize edit set image "ghcr.io/${OWNER}/ahc-app=ghcr.io/${OWNER}/ahc-app:sha-${{ github.sha }}"
+
+      - name: Commit and push
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add kubernetes/overlays/home/kustomization.yaml
+          if git diff --cached --quiet; then
+            echo "No image tag change to commit."
+            exit 0
+          fi
+          git commit -m "chore(deploy): bump ahc-app to sha-${{ github.sha }} [skip ci]"
+          git push
+
+  deploy-workflow:
+    name: Submit deploy-smoke Argo Workflow
+    runs-on: ubuntu-latest
+    needs: bump
+    # Self-disabled until an Argo Server is reachable from CI (self-hosted
+    # runner or tunnel). ArgoCD auto-sync remains the primary deploy path.
+    if: github.event_name == 'push' && vars.ARGO_SERVER_URL != ''
+    steps:
+      - name: Submit workflow via Argo Server API
+        run: |
+          curl -fsS -X POST \
+            -H "Authorization: Bearer ${{ secrets.ARGO_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            "${{ vars.ARGO_SERVER_URL }}/api/v1/workflows/argo/submit" \
+            -d '{
+              "resourceKind": "WorkflowTemplate",
+              "resourceName": "ahc-deploy-smoke",
+              "submitOptions": {
+                "parameters": ["image-tag=sha-${{ github.sha }}"]
+              }
+            }'

.gitignore

@@ -65,6 +65,10 @@ CLAUDE.md
 /tars/
 /kubernetes/**/*.tar
 secret.yaml
+# Filled plain Secrets (templates stay tracked; SealedSecrets in sealed/ stay tracked)
+kubernetes/overlays/*/secrets/*.yaml
+# Fetched sealed-secrets controller certificates
+*.pem
 
 # Volumes
 /db/

README.md

@@ -81,11 +81,16 @@ The stack exposes: Django app on `:8000`, Flower (Celery monitor) on `:5555`.
    uv run python manage.py runserver
    ```
 
-### Kubernetes Deploy (alternative)
+### Kubernetes Deploy (GitOps via ArgoCD)
 
-An alternative to Docker Compose for production-like environments.
-See [`kubernetes/`](kubernetes/) for kustomization files and secret templates.
-Build and load images, then apply with `kubectl apply -k kubernetes/`.
+Kubernetes manifests live in `kubernetes/` as a kustomize base with three overlays
+(`minikube-local` for a plain `kubectl apply -k` dev loop, `minikube-argocd` for the GitOps
+rehearsal, `home` for the production k3s cluster synced by ArgoCD). Images are pulled from
+GHCR; the deployed tag is committed to git by CI.
+
+- Operational runbook (local run, sealing secrets, rehearsal, home bootstrap, backups):
+  [`kubernetes/README.md`](kubernetes/README.md)
+- Architecture decision record: [`doc/13_adr_gitops_argocd.md`](doc/13_adr_gitops_argocd.md)
 
 ## Testing
 

argocd/ahc-home.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ahc-home
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git
+    targetRevision: main
+    path: kubernetes/overlays/home
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ahc
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 3
+      backoff:
+        duration: 30s
+        factor: 2
+        maxDuration: 5m

argocd/ahc-minikube-test.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ahc-minikube-test
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git
+    targetRevision: feat/argocd
+    path: kubernetes/overlays/minikube-argocd
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ahc
+  # Deliberately no automated sync policy: syncs are manual, without prune
+  # and without selfHeal, until drift/prune/sealed-secret-recovery tests
+  # pass on minikube. Automation belongs to ahc-home after the merge.
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

argocd/ahc-workflows.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ahc-workflows
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git
+    targetRevision: main
+    path: workflows
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argo
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

argocd/argo-workflows.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argo-workflows
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://argoproj.github.io/argo-helm
+    chart: argo-workflows
+    targetRevision: 0.45.11
+    helm:
+      values: |
+        server:
+          # Homelab-grade auth: the server acts with its own service account.
+          authModes:
+            - server
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argo
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

argocd/sealed-secrets.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: sealed-secrets
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://bitnami-labs.github.io/sealed-secrets
+    chart: sealed-secrets
+    targetRevision: 2.17.3
+    helm:
+      values: |
+        # Default name expected by the kubeseal CLI discovery.
+        fullnameOverride: sealed-secrets-controller
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

doc/13_adr_gitops_argocd.md

@@ -0,0 +1,93 @@
+## GitOps deployment with ArgoCD
+
+### Date:
+`2026-07-16`
+
+### Status
+In-building
+
+### Context
+Production deployment is pull-based Watchtower polling the `prod` image tag on a docker-compose stack.
+The `kubernetes/` manifests were a leftover from a local minikube workflow: `imagePullPolicy: Never`,
+image tarballs committed under `kubernetes/tars/`, no probes, no namespace, broken Service selectors,
+ephemeral Postgres storage, plaintext secret files (git-ignored) and no way to run Django migrations
+inside the cluster. Deploying that state through any GitOps tool would faithfully reproduce every bug.
+
+The goal is a declarative deployment where git is the single source of truth, the target being a
+single-node k3s cluster on the home server, with local minikube used for rehearsal.
+
+### Decision
+1. **ArgoCD** syncs `kubernetes/overlays/home` from `main`. CI never talks to the cluster; it pushes
+   the image to GHCR and commits a kustomize image-tag bump (`sha-<commit>`) to the home overlay,
+   serialized by a GitHub Actions concurrency group. ArgoCD picks up the git diff.
+2. **Kustomize layout**: `kubernetes/base` plus three overlays:
+   - `minikube-local` — plain git-ignored Secrets, `kubectl apply -k` only (never an ArgoCD source,
+     because ArgoCD checks out the repo and the ignored files are absent there);
+   - `minikube-argocd` — SealedSecrets encrypted for the local minikube controller, used by the
+     manual-sync `ahc-minikube-test` Application to rehearse the full GitOps loop;
+   - `home` — SealedSecrets encrypted for the home cluster, CI-managed image tag, backups, smoke test.
+3. **Sync phasing**: ArgoCD orders resources by phase first and only then by wave, so a PreSync
+   migration hook would run before the databases exist on a fresh cluster. Migrations and CouchDB
+   database creation are therefore **Sync-phase hooks at wave -1**, after ConfigMaps/Secrets/PVCs
+   (wave -3) and the data stores (wave -2), before the app workloads (wave 0) and Ingress (wave 1).
+   The smoke test is PostSync. Outside ArgoCD the jobs guard themselves with wait initContainers.
+4. **Secrets**: Bitnami **Sealed Secrets**. Only true secrets are encrypted (`ahc-app-secrets`,
+   `postgres-credentials`, `couchdb-credentials`, `flower-secrets`); everything else lives in the
+   `ahc-app-config` ConfigMap for readable diffs. Credentials are never duplicated across secrets —
+   the app maps `DB_*` variables from `postgres-credentials` via `secretKeyRef`. A SealedSecret is
+   bound to one controller's private key: minikube and home have separate `sealed/` directories.
+5. **Probes**: split endpoints. `/livez` has no dependencies (liveness + startup), `/readyz` runs
+   `SELECT 1` against Postgres (readiness). CouchDB is deliberately outside `/readyz`: its outage
+   degrades attachments, not the whole application.
+6. **Data stores**: StatefulSets with volumeClaimTemplates. Postgres 18 mounts the volume at
+   `/var/lib/postgresql` (the image keeps PGDATA in a versioned subdirectory since 18). Application
+   file storage gets its own PVCs: `app-media-pvc` (upload dirs mounted via subPath so committed
+   icons stay visible) and `app-private-pvc` shared by web and the Celery worker (offline snapshots).
+7. **Migrations as a Job**, `makemigrations` removed from every deploy path; `collectstatic` runs in
+   a web-pod initContainer writing to an emptyDir shared with gunicorn (ManifestStaticFilesStorage).
+8. **CI → cluster reachability**: GitHub-hosted runners cannot reach the home server, so the deploy
+   signal is ArgoCD polling git (option A). The `ahc-deploy-smoke` Argo WorkflowTemplate is committed
+   for manual runs, and the CI submit step is gated on `vars.ARGO_SERVER_URL` (self-disabled until a
+   self-hosted runner or tunnel exists). The workflow first waits until the Deployment references
+   the expected `image-tag` — otherwise a CI-triggered run could bless a stale rollout that ArgoCD
+   has not synced yet.
+9. **Data-loss guards**: the `ahc` Namespace carries `Prune=confirm,Delete=confirm` and the data
+   PVCs (`app-media-pvc`, `app-private-pvc`, `backup-data`) carry `Prune=false,Delete=false`, so
+   automated prune cannot silently remove them. The app image runs as uid 1000, and hostPath-backed
+   provisioners ignore `fsGroup`, so web/celery pods run a root `prepare-storage` initContainer
+   (mkdir + guarded chown) before the app touches the volumes.
+
+Rejected alternatives: Helm (kustomize already in place, no templating need), ArgoCD Image Updater
+(extra controller; CI commit is simpler and auditable), External Secrets Operator (requires an
+external secret store), SOPS/KSOPS (requires an ArgoCD plugin; kubeseal is plain kubectl-side).
+
+### Consequences
+- Git history becomes the deployment history; rollback is `git revert` of a bump commit.
+- The Watchtower compose stack keeps running during the transition. Two live Celery beat schedulers
+  would duplicate Discord notifications, so the home overlay pins `celery-beat` to 0 replicas until
+  cutover.
+- The first sync starts with **empty databases** — migrating data from the compose stack is a
+  separate manual dump/restore operation.
+- Backups: nightly `pg_dump`, a CouchDB **diagnostic export** (`_all_docs`; a real backup is one-way
+  replication or a PVC snapshot — follow-up), and tar archives of the file PVCs. All land on a PVC on
+  the same disk, which protects against user error, not disk failure; shipping copies off-host
+  (restic/rclone to a NAS) is a post-testing follow-up.
+- Application PVCs are ReadWriteOnce, which is fine on a single node; a multi-node cluster would need
+  RWX or object storage.
+- Losing the sealed-secrets controller key (cluster rebuild) invalidates all SealedSecrets; the
+  git-ignored plain files (kept in a password manager) are the re-sealing source.
+
+### Keywords
+- ArgoCD,
+- GitOps,
+- kustomize,
+- sealed-secrets,
+- sync-wave,
+- k3s.
+
+### Links
+- [ADR-08](08_adr_databases.md) — database responsibilities (PostgreSQL / CouchDB / Redis).
+- [ADR-12](12_adr_turso_offline_snapshots.md) — offline snapshots stored under PRIVATE_STORAGE_ROOT.
+- [kubernetes/README.md](../kubernetes/README.md) — operational runbook.
+- [ArgoCD sync phases and waves](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-waves/).
+- [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets).