Helm chart, GitHub deploy workflow, and Kubernetes/Compose parity

.env.example

@@ -13,7 +13,7 @@ DATAVERSE_DB_NAME=dataverse
 
 SOLR_SERVICE_HOST=solr
 SOLR_SERVICE_PORT=8983
-# Used by init.d scripts inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme.
+# Used by scripts/init.d inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme.
 DATAVERSE_URL=localhost:8080
 DATAVERSE_SERVICE_HOST=localhost.direct
 # Optional: admin API token for branding/seed compose jobs (normally secrets/api/key is filled from bootstrap.env by dev_branding / dev_seed entrypoints).
@@ -26,7 +26,7 @@ POSTGRES_DATABASE=dataverse
 POSTGRES_DB=dataverse
 POSTGRES_PORT=5432
 
-# Public hostname (init.d sets dataverse.siteUrl to https://<hostname> — use Stack Car *.localhost.direct).
+# Public hostname (scripts/init.d sets dataverse.siteUrl to https://<hostname> — use Stack Car *.localhost.direct).
 hostname=localhost.direct
 # Traefik Host() rules use ${traefikhost} and subdomains (solr.<traefikhost>, minio.<traefikhost>, …).
 traefikhost=localhost.direct
@@ -39,22 +39,35 @@ MINIO_ROOT_PASSWORD=love1234
 # Optional; unused for local dev when TLS is provided by `sc proxy` (Stack Car).
 useremail=you@example.org
 
+# --- Optional: SMTP (scripts/init.d/010-mailrelay-set.sh; requires system_email; smtp_enabled not false/0/no) ---
+# smtp_enabled=true
+# smtp_type=plain
+# smtp_auth=true
+# smtp_starttls=true
+# smtp_port=587
+# socket_port=587
+# mailhost=smtp.example.com
+# mailuser=apikey-or-username
+# smtp_password=secret
+# system_email=support@example.com
+# no_reply_email=noreply@example.com
+
 WEBHOOK=/opt/payara/triggers/external-services.py
 
-# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / init.d)
+# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / scripts/init.d)
 dataciterestapiurlstring=https\\:\/\/api.test.datacite.org
 baseurlstring=https\:\/\/mds.test.datacite.org
 
-# Optional builtin-user API key override (see init.d/01-persistent-id.sh)
+# Optional builtin-user API key override (see scripts/init.d/01-persistent-id.sh)
 # BUILTIN_USERS_KEY=burrito
 
-# --- Optional: MinIO buckets (only if init.d minio scripts should run) ---
+# --- Optional: MinIO buckets (only if scripts/init.d minio scripts should run) ---
 # minio_label_1=
 # bucketname_1=
 # minio_bucket_1=
 # minio_profile_1=
 
-# --- Optional: AWS S3 store (only if init.d/006-s3-aws-storage.sh should run) ---
+# --- Optional: AWS S3 store (only if scripts/init.d/006-s3-aws-storage.sh should run) ---
 # aws_bucket_name=
 # aws_endpoint_url=
 # aws_s3_profile=

.github/workflows/deploy.yaml

@@ -27,7 +27,7 @@ on:
         required: false
         type: string
       run_bootstrap_job:
-        description: Re-run the bootstrap Job (helmHook=false) — same manifest as chart templates/bootstrap-job.yaml; use if the post-install hook was skipped or you reset Postgres. Chart default mode is oneShot (configbaker bootstrap.sh dev). Deletes the prior Job by name first.
+        description: Re-run the bootstrap Job (helmHook=false, compose). Requires Secret dataverse-admin-api-token (key token) in the namespace when the instance is already bootstrapped — see ops/*-deploy.tmpl.yaml bootstrapJob.compose.existingAdminApiTokenSecret. Deletes the prior Job by name first.
         required: false
         type: boolean
         default: false
@@ -42,20 +42,8 @@ jobs:
     environment: ${{ inputs.environment }}
     env:
       DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
-      # Optional mail — secrets + Environment variables (see ops/demo-deploy.tmpl.yaml header).
-      SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }}
-      NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }}
+      # Mail: SendGrid API key only — host/ports/From/support address are literals in ops/<env>-deploy.tmpl.yaml.
       SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
-      MAIL_SMTP_PASSWORD: ${{ secrets.MAIL_SMTP_PASSWORD }}
-      SMTP_ADDRESS: ${{ vars.SMTP_ADDRESS }}
-      SMTP_USER_NAME: ${{ vars.SMTP_USER_NAME }}
-      SMTP_PORT: ${{ vars.SMTP_PORT }}
-      SOCKET_PORT: ${{ vars.SOCKET_PORT }}
-      SMTP_AUTH: ${{ vars.SMTP_AUTH }}
-      SMTP_STARTTLS: ${{ vars.SMTP_STARTTLS }}
-      SMTP_TYPE: ${{ vars.SMTP_TYPE }}
-      SMTP_ENABLED: ${{ vars.SMTP_ENABLED }}
-      SMTP_DOMAIN: ${{ vars.SMTP_DOMAIN }}
       DEPLOY_ENVIRONMENT: ${{ inputs.environment }}
       HELM_RELEASE_NAME: ${{ inputs.k8s_release_name || format('{0}-{1}', inputs.environment, github.event.repository.name) }}
       HELM_NAMESPACE: ${{ inputs.k8s_namespace || format('{0}-{1}', inputs.environment, github.event.repository.name) }}
@@ -94,16 +82,19 @@ jobs:
           TMPL="ops/${DEPLOY_ENVIRONMENT}-deploy.tmpl.yaml"
           OUT="ops/${DEPLOY_ENVIRONMENT}-deploy.yaml"
           echo "$KUBECONFIG_FILE" | base64 -d >"$KUBECONFIG"
-          export SMTP_PORT="${SMTP_PORT:-25}"
-          export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}"
-          export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}"
-          if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then
-            export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}"
-          fi
-          # Only secrets + rollout id — hosts, Solr DNS, ingress, and bucket are literals in the *.tmpl.yaml file.
-          ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH'
+          # envsubst: only secrets + rollout id. Everything else (SMTP host, ports, From, URLs, Solr, …) is literals in the *.tmpl.yaml file.
+          ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SMTP_PASSWORD'
           envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT"
 
+      - name: Solr conf ConfigMap (pre-Helm)
+        env:
+          DV_REF: ${{ vars.DV_REF || 'v6.10.1' }}
+          SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }}
+        run: |
+          set -e
+          chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh
+          SOLR_RESTART_DEPLOYMENTS=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME"
+
       - name: Deploy with Helm
         run: |
           set -e
@@ -116,6 +107,16 @@ jobs:
             -l "app.kubernetes.io/instance=${HELM_RELEASE_NAME},app.kubernetes.io/name=${HELM_APP_NAME}" \
             --timeout="${DEPLOY_ROLLOUT_TIMEOUT}"
 
+      - name: Solr workloads pick up ConfigMap updates
+        if: ${{ vars.SOLR_POST_HELM_ROLLOUT == 'true' }}
+        env:
+          DV_REF: ${{ vars.DV_REF || 'v6.10.1' }}
+          SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }}
+        run: |
+          set -e
+          chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh
+          SOLR_APPLY_CM=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME"
+
       - name: One-shot bootstrap Job (non-hook)
         if: github.event_name == 'workflow_dispatch' && inputs.run_bootstrap_job
         run: |
@@ -125,9 +126,11 @@ jobs:
           helm template "$HELM_RELEASE_NAME" "$HELM_CHART_PATH" \
             --namespace "$HELM_NAMESPACE" \
             $HELM_EXTRA_ARGS \
+            --show-only templates/bootstrap-chain-configmap.yaml \
             --show-only templates/bootstrap-job.yaml \
             --set bootstrapJob.enabled=true \
             --set bootstrapJob.helmHook=false \
+            --set bootstrapJob.mode=compose \
             >"$OUT"
           JOB="$(awk '/^kind: Job$/{j=1} j && /^  name: /{print $2; exit}' "$OUT")"
           if [ -z "$JOB" ]; then
@@ -137,4 +140,4 @@ jobs:
           kubectl -n "$HELM_NAMESPACE" delete job "$JOB" --ignore-not-found=true
           kubectl apply -f "$OUT"
           kubectl -n "$HELM_NAMESPACE" wait --for=condition=complete "job/$JOB" --timeout="${DEPLOY_BOOTSTRAP_JOB_TIMEOUT}"
-          kubectl -n "$HELM_NAMESPACE" logs "job/$JOB"
+          kubectl -n "$HELM_NAMESPACE" logs "job/$JOB"

.gitignore

@@ -12,4 +12,4 @@ minio-data/
 data/
 /docroot/
 *.war
-.idea/
+.idea/

README.md

@@ -1,6 +1,11 @@
 # DataverseUp
 
-Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and room to grow toward hosted AWS/Kubernetes without forking core.
+Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and a **Helm chart** for Kubernetes without forking core.
+
+| | |
+|--|--|
+| **Repository** | [github.com/notch8/dataverseup](https://github.com/notch8/dataverseup) |
+| **Live demo** | [demo-dataverseup.notch8.cloud](https://demo-dataverseup.notch8.cloud/) — deployed from this stack (Helm/Kubernetes); seeded demo data for smoke tests |
 
 ## Quick start (local / lab)
 
@@ -46,22 +51,28 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G
    ```
    If you need a manual token, put it on one line in **`secrets/api/key`** (superuser token from the UI), then run **`dev_branding`** again.
 
+## Kubernetes (Helm)
+
+- **Chart:** `charts/dataverseup` — see **[charts/dataverseup/README.md](charts/dataverseup/README.md)** for a feature summary and `helm` commands.
+- **Runbook:** **[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md)** — Helm (prereqs, Secrets, Solr, S3, bootstrap), GitHub Actions deploy, `./bin/helm_deploy`, smoke tests, upgrades.
+
+Compose remains the default path for local/lab; Helm reuses the same Payara scripts from **`scripts/init.d/`** (chart symlinks under `charts/dataverseup/files/init.d/`) where applicable.
+
 ## Layout
 
 | Path | Purpose |
 |------|---------|
 | `docker-compose.yml` | Stack: Postgres, Solr, MinIO (optional), Dataverse, bootstrap, branding, seed; Traefik labels + **`networks.default.name: stackcar`** (Stack Car proxy) |
 | `.env.example` | Version pins and env template — copy to `.env` |
 | `secrets.example/` | Payara/Dataverse secret files template — copy to **`secrets/`** (see Quick start) |
-| `init.d/` | Payara init scripts (local storage, optional S3/MinIO when env set) |
-| `init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` |
-| `config/schema.xml`, `config/solrconfig.xml` | Solr conf bind-mounts / upstream copies (see `scripts/solr-initdb/`) |
-| `config/update-fields.sh` | Upstream metadata-block tooling helper |
+| `scripts/` | **All automation:** Payara **`init.d/`**, Compose/K8s entrypoints, **`apply-branding.sh`**, **`seed-content.sh`**, **`solr-initdb/`**, **`solr/update-fields.sh`**, **`triggers/`**, **`k8s/`** |
+| `scripts/init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` |
+| `config/schema.xml`, `config/solrconfig.xml` | Solr XML bind-mounts / upstream copies (see `scripts/solr-initdb/`) |
 | `branding/` | Installation branding + static assets |
 | `fixtures/seed/` | JSON + files for **`dev_seed`** |
-| `scripts/` | Bootstrap, branding, seed entrypoints, `apply-branding.sh`, `solr-initdb/` |
-| `triggers/` | Postgres notify + optional webhook script (see **`WEBHOOK`** in `.env.example`) |
-| `docs/DEPLOYMENT.md` | **Working deployment notes + learnings** (add in-repo when you maintain runbooks) |
+| `charts/dataverseup/` | Helm chart for Dataverse on Kubernetes (optional Solr, bootstrap Job, S3, Ingress, …) |
+| `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/DEPLOYMENT.md`**) |
+| `docs/DEPLOYMENT.md` | **Deployment runbook:** Helm/Kubernetes, optional GitHub Actions, Compose pointer, learnings log |
 
 ## Version pin
 
@@ -73,8 +84,8 @@ Compose uses **`solr:9.10.1`** with IQSS **`schema.xml`** / **`solrconfig.xml`**
 REF=develop  # or a release tag, e.g. v6.10.1
 curl -fsSL -o config/schema.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/schema.xml"
 curl -fsSL -o config/solrconfig.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/solrconfig.xml"
-curl -fsSL -o config/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh"
-chmod +x config/update-fields.sh
+curl -fsSL -o scripts/solr/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh"
+chmod +x scripts/solr/update-fields.sh
 ```
 
 If you previously ran Solr 8, remove the compose Solr volume once so the core is recreated under Solr 9, then reindex from Dataverse.
@@ -88,4 +99,4 @@ If you previously ran Solr 8, remove the compose Solr volume once so the core is
 
 ## License
 
-Dataverse is licensed by IQSS; container images by their publishers.
+Dataverse is licensed by IQSS; container images by their publishers.

bin/helm_deploy

@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# Helm upgrade/install for the Dataverse chart (from repository root).
+# Chart path defaults to ./charts/dataverseup; override with HELM_CHART_PATH.
+# Pass extra flags via HELM_EXTRA_ARGS, e.g.:
+#   HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace
+#
+# A second --timeout in HELM_EXTRA_ARGS overrides the default below (Helm uses the last value).
+
+set -e
+
+if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then
+  echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2
+  echo "  Run from the repository root (HELM_CHART_PATH defaults to ./charts/dataverseup)." >&2
+  exit 1
+fi
+
+release_name="$1"
+namespace="$2"
+chart_path="${HELM_CHART_PATH:-./charts/dataverseup}"
+
+helm upgrade \
+  --install \
+  --atomic \
+  --timeout 30m0s \
+  ${HELM_EXTRA_ARGS:-} \
+  --namespace="$namespace" \
+  --create-namespace \
+  "$release_name" \
+  "$chart_path"

branding.env

@@ -0,0 +1 @@
+branding/branding.env

branding/branding.env

@@ -3,7 +3,7 @@
 # For :HeaderCustomizationFile / :FooterCustomizationFile / :StyleCustomizationFile use *filesystem* paths in the
 # API, e.g. /dv/docroot/branding/custom-header.html (see branding.env examples below).
 
-# INSTALLATION_NAME='My Repository'
+INSTALLATION_NAME='DataverseUp'
 
 # Navbar logo file on disk: branding/docroot/logos/navbar/logo.svg → URL below (must match filename).
 LOGO_CUSTOMIZATION_FILE=/logos/navbar/logo.svg
@@ -21,8 +21,8 @@ DISABLE_ROOT_DATAVERSE_THEME=true
 
 # Multi-word values must be quoted. FOOTER_COPYRIGHT is appended to the built-in "Copyright © YEAR" with no
 # separator in the UI; apply-branding.sh prepends a space unless you start with space/tab, —, -, |, or (.
-FOOTER_COPYRIGHT='Notch8'
-# NAVBAR_ABOUT_URL=https://example.edu/about
-# NAVBAR_SUPPORT_URL=mailto:support@example.edu
-# NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/
+FOOTER_COPYRIGHT=' DataverseUp | Hosted by <a href="https://notch8.com" rel="noopener noreferrer">Notch8</a>, your partner in digital preservation.'
 
+NAVBAR_ABOUT_URL=https://notch8.com/about
+NAVBAR_SUPPORT_URL=mailto:support@notch8.com
+NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/

charts/dataverseup/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/dataverseup/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: dataverseup
+description: Dataverse (Payara) on Kubernetes; optional in-cluster standalone Solr; Postgres external; optional S3.
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.12
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "6.10.1-noble-r0"

charts/dataverseup/README.md

@@ -0,0 +1,38 @@
+# dataverseup Helm chart
+
+Deploys **stock Dataverse** (GDCC `gdcc/dataverse` image) on Kubernetes with optional:
+
+- **Persistent** file store and optional **docroot** PVC (branding / logos)
+- **Bootstrap** Job (`gdcc/configbaker`)
+- **Internal Solr** + **solrInit** initContainer
+- **S3** storage driver (AWS-style credentials Secret)
+- **Ingress**, **HPA**, **ServiceAccount**
+
+## Quick commands
+
+```bash
+helm lint charts/dataverseup
+helm template release-name charts/dataverseup -f your-values.yaml
+helm upgrade --install release-name charts/dataverseup -n your-namespace -f your-values.yaml
+```
+
+## Documentation
+
+See **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)** in this repository for prerequisites, Secret layout, smoke tests, and CI deploy notes.
+
+## Payara init scripts (S3, mail)
+
+S3 and mail relay scripts (and the full Payara bundle when `initdFromChart.enabled`) live under **`files/init.d/`** as **symbolic links** to **`scripts/init.d/`** (same tree Docker Compose mounts at `/opt/payara/init.d`). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained.
+
+## Configuration
+
+| Key | Purpose |
+|-----|---------|
+| `image.repository` / `image.tag` | GDCC Dataverse image |
+| `extraEnvFrom` / `extraEnvVars` | DB, Solr, URL, JVM — **use Secrets for credentials** |
+| `persistence` | RWO PVC for `/data` |
+| `internalSolr` + `solrInit` | **Dedicated Solr 9** pod with this release (not a shared cluster Solr). Default **`solrInit.mode: standalone`**; empty **`solrInit.solrHttpBase`** → chart uses the in-release Solr Service. Core **`dataverse`** (Compose uses **`collection1`**). SolrCloud (`mode: cloud`) needs ZK + **full** conf or `solr-conf.tgz`. |
+| `bootstrapJob` | First-time `configbaker` bootstrap |
+| `ingress` | HTTP routing to Service port 80 |
+
+Solr alignment with Docker Compose (IQSS **`config/`** files, core naming, `solrInit` overrides) is documented in **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)**.

charts/dataverseup/files/apply-branding.sh

@@ -0,0 +1 @@
+../../../scripts/apply-branding.sh

charts/dataverseup/files/branding-navbar-logo.svg

@@ -0,0 +1 @@
+../../../branding/docroot/logos/navbar/logo.svg

charts/dataverseup/files/branding.env

@@ -0,0 +1 @@
+../../../branding.env

charts/dataverseup/files/fixtures-seed/dataset-images.json

@@ -0,0 +1 @@
+../../../../fixtures/seed/dataset-images.json

charts/dataverseup/files/fixtures-seed/dataset-tabular.json

@@ -0,0 +1 @@
+../../../../fixtures/seed/dataset-tabular.json

charts/dataverseup/files/fixtures-seed/demo-collection.json

@@ -0,0 +1 @@
+../../../../fixtures/seed/demo-collection.json

charts/dataverseup/files/fixtures-seed/files_1x1.png

@@ -0,0 +1 @@
+../../../../fixtures/seed/files/1x1.png

charts/dataverseup/files/fixtures-seed/files_badge.svg

@@ -0,0 +1 @@
+../../../../fixtures/seed/files/badge.svg

charts/dataverseup/files/fixtures-seed/files_readme.txt

@@ -0,0 +1 @@
+../../../../fixtures/seed/files/readme.txt