From 1b2d568f0118c7596da23b8facd77b245dfab0a7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 1 Apr 2026 22:24:02 -0700 Subject: [PATCH 01/31] Add Helm chart charts/dataverseup (Dataverse 6.10 GDCC) - Port chart from internal reference; rename templates to dataverseup - Chart version 0.1.0; scrub internal ops path references from values/messages - Fix helm test hook to wget /api/info/version over HTTP - docs/HELM.md install order, docs/DEPLOYMENT.md ticket context - values-examples/internal-solr-starter.yaml skeleton - README: Helm quick start and layout Made-with: Cursor --- README.md | 15 +- charts/dataverseup/.helmignore | 23 ++ charts/dataverseup/Chart.yaml | 24 ++ charts/dataverseup/README.md | 34 +++ .../dataverseup/files/006-s3-aws-storage.sh | 66 +++++ charts/dataverseup/files/010-mailrelay-set.sh | 33 +++ charts/dataverseup/templates/NOTES.txt | 29 ++ charts/dataverseup/templates/_helpers.tpl | 92 ++++++ .../dataverseup/templates/bootstrap-job.yaml | 44 +++ charts/dataverseup/templates/configmap.yaml | 22 ++ charts/dataverseup/templates/deployment.yaml | 223 +++++++++++++++ charts/dataverseup/templates/hpa.yaml | 32 +++ charts/dataverseup/templates/ingress.yaml | 61 ++++ .../templates/internal-solr-deployment.yaml | 112 ++++++++ .../templates/internal-solr-pvc.yaml | 17 ++ .../templates/internal-solr-service.yaml | 17 ++ charts/dataverseup/templates/pvc-docroot.yaml | 18 ++ charts/dataverseup/templates/pvc.yaml | 17 ++ charts/dataverseup/templates/service.yaml | 15 + .../dataverseup/templates/serviceaccount.yaml | 13 + .../templates/solr-init-configmap.yaml | 140 +++++++++ .../templates/tests/test-connection.yaml | 17 ++ .../internal-solr-starter.yaml | 67 +++++ charts/dataverseup/values.yaml | 269 ++++++++++++++++++ docs/DEPLOYMENT.md | 22 ++ docs/HELM.md | 99 +++++++ 26 files changed, 1519 insertions(+), 2 deletions(-) create mode 100644 charts/dataverseup/.helmignore create mode 100644 charts/dataverseup/Chart.yaml create mode 100644 charts/dataverseup/README.md create mode 100644 charts/dataverseup/files/006-s3-aws-storage.sh create mode 100644 charts/dataverseup/files/010-mailrelay-set.sh create mode 100644 charts/dataverseup/templates/NOTES.txt create mode 100644 charts/dataverseup/templates/_helpers.tpl create mode 100644 charts/dataverseup/templates/bootstrap-job.yaml create mode 100644 charts/dataverseup/templates/configmap.yaml create mode 100644 charts/dataverseup/templates/deployment.yaml create mode 100644 charts/dataverseup/templates/hpa.yaml create mode 100644 charts/dataverseup/templates/ingress.yaml create mode 100644 charts/dataverseup/templates/internal-solr-deployment.yaml create mode 100644 charts/dataverseup/templates/internal-solr-pvc.yaml create mode 100644 charts/dataverseup/templates/internal-solr-service.yaml create mode 100644 charts/dataverseup/templates/pvc-docroot.yaml create mode 100644 charts/dataverseup/templates/pvc.yaml create mode 100644 charts/dataverseup/templates/service.yaml create mode 100644 charts/dataverseup/templates/serviceaccount.yaml create mode 100644 charts/dataverseup/templates/solr-init-configmap.yaml create mode 100644 charts/dataverseup/templates/tests/test-connection.yaml create mode 100644 charts/dataverseup/values-examples/internal-solr-starter.yaml create mode 100644 charts/dataverseup/values.yaml create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/HELM.md diff --git a/README.md b/README.md index 9b6c5bb..b267f3b 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G 5. **URLs (defaults in `.env.example`):** - Dataverse (Traefik): `http://localhost/` - Direct Payara: `http://localhost:8080/` - - Bootstrap admin (after `dev_bootstrap` succeeds): **`dataverseAdmin`** / **`admin1`** (change before any shared or AWS host; see `docs/DEPLOYMENT.md`) + - Bootstrap admin (after `dev_bootstrap` succeeds): **`dataverseAdmin`** / **`admin1`** 6. **Branding (optional):** After creating a **superuser API token** in the UI, put it on one line in `secrets/api/key`, then: ```bash @@ -35,18 +35,29 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G ``` Or: `./scripts/dev-up.sh` (brings stack up and re-runs branding). +## Kubernetes (Helm) + +```bash +helm lint charts/dataverseup +helm upgrade --install dataverseup charts/dataverseup -n -f your-values.yaml +``` + +Full steps, Secrets, and Solr ConfigMap expectations: **[docs/HELM.md](docs/HELM.md)**. Example values skeleton: **`charts/dataverseup/values-examples/internal-solr-starter.yaml`**. + ## Layout | Path | Purpose | |------|---------| | `docker-compose.yml` | Stack: Traefik, Postgres, Solr, MinIO (optional), Dataverse, bootstrap, branding | +| `charts/dataverseup/` | **Helm chart** for Kubernetes | | `.env.example` | Version pins and env template — copy to `.env` | | `secrets.example/` | Payara/Dataverse secret files template — copy to `secrets/` | | `init.d/` | Payara init scripts (local storage, optional S3/MinIO when env set) | | `config/schema.xml` | Solr schema bind-mount (see upstream Solr notes) | | `branding/` | Installation branding + static assets | | `scripts/` | Helpers (`apply-branding.sh`, `dev-up.sh`) | -| `docs/DEPLOYMENT.md` | **Working deployment notes + learnings** | +| `docs/HELM.md` | **Helm install / smoke tests / learnings** | +| `docs/DEPLOYMENT.md` | **Working deployment notes (Compose + AWS context)** | ## Version pin diff --git a/charts/dataverseup/.helmignore b/charts/dataverseup/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/dataverseup/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/dataverseup/Chart.yaml b/charts/dataverseup/Chart.yaml new file mode 100644 index 0000000..a287a57 --- /dev/null +++ b/charts/dataverseup/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: dataverseup +description: Dataverse (Payara) on Kubernetes; optional in-cluster standalone Solr; Postgres external; optional S3. + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "6.10.1-noble-r0" diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md new file mode 100644 index 0000000..ebd67dd --- /dev/null +++ b/charts/dataverseup/README.md @@ -0,0 +1,34 @@ +# dataverseup Helm chart + +Deploys **stock Dataverse** (GDCC `gdcc/dataverse` image) on Kubernetes with optional: + +- **Persistent** file store and optional **docroot** PVC (branding / logos) +- **Bootstrap** Job (`gdcc/configbaker`) +- **Internal Solr** + **solrInit** initContainer +- **S3** storage driver (AWS-style credentials Secret) +- **Ingress**, **HPA**, **ServiceAccount** + +## Quick commands + +```bash +helm lint charts/dataverseup +helm template release-name charts/dataverseup -f your-values.yaml +helm upgrade --install release-name charts/dataverseup -n your-namespace -f your-values.yaml +``` + +## Documentation + +See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, Secret layout, and smoke tests. + +## Configuration + +| Key | Purpose | +|-----|---------| +| `image.repository` / `image.tag` | GDCC Dataverse image | +| `extraEnvFrom` / `extraEnvVars` | DB, Solr, URL, JVM — **use Secrets for credentials** | +| `persistence` | RWO PVC for `/data` | +| `internalSolr` + `solrInit` | In-cluster Solr; requires **full** Solr conf ConfigMap | +| `bootstrapJob` | First-time `configbaker` bootstrap | +| `ingress` | HTTP routing to Service port 80 | + +Example skeleton: **`values-examples/internal-solr-starter.yaml`**. diff --git a/charts/dataverseup/files/006-s3-aws-storage.sh b/charts/dataverseup/files/006-s3-aws-storage.sh new file mode 100644 index 0000000..beee273 --- /dev/null +++ b/charts/dataverseup/files/006-s3-aws-storage.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# AWS S3 file store for Dataverse. +# https://guides.dataverse.org/en/latest/installation/config.html#s3-storage +# +# Sourced by the base image entrypoint from /opt/payara/scripts/init.d/ *before* Payara starts. +# `asadmin create-jvm-options` cannot run here (DAS is not up on :4848). We append +# `create-system-properties` lines to POSTBOOT_COMMANDS_FILE like init_2_configure.sh. +# +# Requires env: aws_bucket_name, aws_endpoint_url, aws_s3_profile, aws_s3_region (chart sets these when awsS3.enabled). +# Kubernetes: chart sets AWS_SHARED_CREDENTIALS_FILE / AWS_CONFIG_FILE (mounted Secret). +# Compose (often root): copy credentials into ~/.aws when those vars are unset. + +if [ -n "${aws_bucket_name:-}" ]; then + if [ -z "${AWS_SHARED_CREDENTIALS_FILE:-}" ] && [ -r /secrets/aws-cli/.aws/credentials ]; then + if mkdir -p /root/.aws 2>/dev/null; then + cp -R /secrets/aws-cli/.aws/. /root/.aws/ + else + _aws_dir="${HOME:-/opt/payara}/.aws" + mkdir -p "$_aws_dir" || { _aws_dir="/opt/payara/.aws" && mkdir -p "$_aws_dir"; } + cp -R /secrets/aws-cli/.aws/. "${_aws_dir}/" + fi + fi + + if [ -z "${POSTBOOT_COMMANDS_FILE:-}" ] || [ ! -w "$POSTBOOT_COMMANDS_FILE" ]; then + echo "006-s3-aws-storage: POSTBOOT_COMMANDS_FILE missing or not writable" >&2 + return 1 + fi + + # Keep dataverse.files.local.* so the "local" driver stays registered for existing DB rows that still + # reference that storage identifier. Set default uploads to S3 via storage-driver-id=S3 only. + # Strip prior S3.* and storage-driver-id lines so pod restarts do not duplicate properties. + _pb_pre=$(mktemp) + _pb_dep=$(mktemp) + trap 'rm -f "${_pb_pre:-}" "${_pb_dep:-}"' EXIT + grep -v -E '^create-system-properties dataverse\.files\.storage-driver-id=' "$POSTBOOT_COMMANDS_FILE" \ + | grep -v -E '^create-system-properties dataverse\.files\.S3\.' \ + | grep -v -E '^deploy ' > "$_pb_pre" || true + grep -E '^deploy ' "$POSTBOOT_COMMANDS_FILE" > "$_pb_dep" || true + _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') + # With custom-endpoint-url set, Dataverse's S3AccessIO uses JVM key custom-endpoint-region for SigV4. + # If unset, upstream defaults to the literal string "dataverse" → S3 400 "region 'dataverse' is wrong". + _s3_reg="${aws_s3_region:-${AWS_REGION:-}}" + if [ -z "${_s3_reg}" ]; then + echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when using custom-endpoint-url" >&2 + return 1 + fi + { + cat "$_pb_pre" + echo "create-system-properties dataverse.files.S3.type=s3" + echo "create-system-properties dataverse.files.S3.label=S3" + echo "create-system-properties dataverse.files.S3.bucket-name=${aws_bucket_name}" + echo "create-system-properties dataverse.files.S3.download-redirect=true" + echo "create-system-properties dataverse.files.S3.url-expiration-minutes=120" + echo "create-system-properties dataverse.files.S3.connection-pool-size=4096" + echo "create-system-properties dataverse.files.storage-driver-id=S3" + echo "create-system-properties dataverse.files.S3.profile=${aws_s3_profile}" + echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" + echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" + cat "$_pb_dep" + } > "$POSTBOOT_COMMANDS_FILE" + trap - EXIT + rm -f "$_pb_pre" "$_pb_dep" + # Payara is not listening yet; set via Admin UI/API after first boot if needed. + curl -sfS -m 2 -X PUT "http://127.0.0.1:8080/api/admin/settings/:DownloadMethods" -d "native/http" 2>/dev/null || true +fi diff --git a/charts/dataverseup/files/010-mailrelay-set.sh b/charts/dataverseup/files/010-mailrelay-set.sh new file mode 100644 index 0000000..1577276 --- /dev/null +++ b/charts/dataverseup/files/010-mailrelay-set.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Setup mail relay (compose-compatible; mounted when values.mail.enabled). +# https://guides.dataverse.org/en/latest/developers/troubleshooting.html +# +# smtp_enabled: false|0|no skips this script (GitHub vars SMTP_ENABLED). +# smtp_type=plain implies SMTP AUTH after STARTTLS if smtp_auth is unset (Rails-style). +# smtp_auth / smtp_starttls: true|1|yes when set explicitly. +case "${smtp_enabled}" in + false|0|no|NO|False) exit 0 ;; +esac + +if [ "${system_email}" ]; then + curl -X PUT -d ${system_email} http://localhost:8080/api/admin/settings/:SystemEmail + asadmin --user=${ADMIN_USER} --passwordfile=${PASSWORD_FILE} delete-javamail-resource mail/notifyMailSession + + AUTH_PROP="mail.smtp.auth=false" + case "${smtp_auth}" in + true|1|yes|TRUE|Yes) AUTH_PROP="mail.smtp.auth=true" ;; + esac + if [ "${AUTH_PROP}" = "mail.smtp.auth=false" ]; then + case "${smtp_type}" in + plain|PLAIN) AUTH_PROP="mail.smtp.auth=true" ;; + esac + fi + + PROPS="${AUTH_PROP}:mail.smtp.password=${smtp_password}:mail.smtp.port=${smtp_port}:mail.smtp.socketFactory.port=${socket_port}:mail.smtp.socketFactory.fallback=false" + case "${smtp_starttls}" in + true|1|yes|TRUE|Yes) PROPS="${PROPS}:mail.smtp.starttls.enable=true" ;; + esac + + asadmin --user=${ADMIN_USER} --passwordfile=${PASSWORD_FILE} create-javamail-resource --mailhost ${mailhost} --mailuser ${mailuser} --fromaddress ${no_reply_email} --property "${PROPS}" mail/notifyMailSession +fi diff --git a/charts/dataverseup/templates/NOTES.txt b/charts/dataverseup/templates/NOTES.txt new file mode 100644 index 0000000..08ec125 --- /dev/null +++ b/charts/dataverseup/templates/NOTES.txt @@ -0,0 +1,29 @@ +{{- if .Values.internalSolr.enabled }} +In-cluster standalone Solr (no ZooKeeper): Service {{ include "dataverseup.fullname" . }}-solr port 8983 — from another pod in this namespace use http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 . Ensure solrInit.solrHttpBase and DATAVERSE_SOLR_HOST match that host. + +{{- end }} +{{- if .Values.docrootPersistence.enabled }} +Docroot / branding: PVC {{ if .Values.docrootPersistence.existingClaim }}{{ .Values.docrootPersistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-docroot{{ end }} mounted at {{ .Values.docrootPersistence.mountPath }} (DATAVERSE_FILES_DOCROOT for /logos/*). +{{- end }} +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "dataverseup.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "dataverseup.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "dataverseup.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "dataverseup.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=primary" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/dataverseup/templates/_helpers.tpl b/charts/dataverseup/templates/_helpers.tpl new file mode 100644 index 0000000..08f4b41 --- /dev/null +++ b/charts/dataverseup/templates/_helpers.tpl @@ -0,0 +1,92 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "dataverseup.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dataverseup.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dataverseup.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dataverseup.labels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +{{ include "dataverseup.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dataverseup.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dataverseup.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service / CLI label query for the main Dataverse pods only. Pods also set component=primary; +Deployment matchLabels stay name+instance only so upgrades do not hit immutable selector changes. +*/}} +{{- define "dataverseup.primarySelectorLabels" -}} +{{ include "dataverseup.selectorLabels" . }} +app.kubernetes.io/component: primary +{{- end }} + +{{/* +Labels for the optional in-chart standalone Solr Deployment/Service (must NOT match dataverseup.selectorLabels +or the main Deployment ReplicaSet will count Solr pods). +*/}} +{{- define "dataverseup.internalSolrLabels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-solr +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: internal-solr +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "dataverseup.internalSolrSelectorLabels" -}} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-solr +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: internal-solr +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "dataverseup.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "dataverseup.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml new file mode 100644 index 0000000..e25871b --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -0,0 +1,44 @@ +{{- if .Values.bootstrapJob.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap{{ if not .Values.bootstrapJob.helmHook }}-once{{ end }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: configbaker-bootstrap + {{- if .Values.bootstrapJob.helmHook }} + annotations: + helm.sh/hook: post-install + helm.sh/hook-weight: "15" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed + {{- end }} +spec: + backoffLimit: {{ .Values.bootstrapJob.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.bootstrapJob.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "dataverseup.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: configbaker-bootstrap + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: configbaker + image: "{{ .Values.bootstrapJob.image.repository }}:{{ .Values.bootstrapJob.image.tag | default .Values.image.tag }}" + imagePullPolicy: {{ .Values.bootstrapJob.image.pullPolicy }} + command: {{- toYaml .Values.bootstrapJob.command | nindent 12 }} + env: + - name: DATAVERSE_URL + value: {{ .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) | quote }} + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + {{- with .Values.bootstrapJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/configmap.yaml b/charts/dataverseup/templates/configmap.yaml new file mode 100644 index 0000000..41bc6a8 --- /dev/null +++ b/charts/dataverseup/templates/configmap.yaml @@ -0,0 +1,22 @@ +{{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-config + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +data: + {{- if .Values.configMap.enabled }} + {{- toYaml .Values.configMap.data | nindent 2 }} + {{- end }} + {{- if .Values.mail.enabled }} + {{- $mailScript := .Files.Get "files/010-mailrelay-set.sh" | trim }} + 010-mailrelay-set.sh: | +{{- $mailScript | nindent 4 }} + {{- end }} + {{- if .Values.awsS3.enabled }} + {{- $s3Script := .Files.Get "files/006-s3-aws-storage.sh" | trim }} + 006-s3-aws-storage.sh: | +{{- $s3Script | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/deployment.yaml b/charts/dataverseup/templates/deployment.yaml new file mode 100644 index 0000000..e8d9845 --- /dev/null +++ b/charts/dataverseup/templates/deployment.yaml @@ -0,0 +1,223 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "dataverseup.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "dataverseup.labels" . | nindent 8 }} + app.kubernetes.io/component: primary + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "dataverseup.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if or .Values.solrInit.enabled .Values.solrPreSetupInitContainer }} + initContainers: + {{- with .Values.solrPreSetupInitContainer }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.solrInit.enabled }} + {{- if and .Values.solrInit.existingSecret .Values.solrInit.adminUser }}{{ fail "solrInit: use either existingSecret or adminUser/adminPassword, not both" }}{{- end }} + - name: load-solr-config + image: "{{ .Values.solrInit.image }}" + imagePullPolicy: {{ .Values.solrInit.imagePullPolicy }} + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + command: ["/bin/bash", "/scripts/solr-init.sh"] + {{- if .Values.solrInit.existingSecret }} + envFrom: + - secretRef: + name: {{ .Values.solrInit.existingSecret | quote }} + {{- end }} + env: + {{- if or .Values.internalSolr.enabled (eq .Values.solrInit.mode "standalone") }} + - name: SOLR_INIT_MODE + value: "standalone" + {{- end }} + - name: SOLR_ZK_CONNECT + value: {{ .Values.solrInit.zkConnect | default "" | quote }} + - name: SOLR_HTTP_BASE + value: {{ .Values.solrInit.solrHttpBase | quote }} + - name: SOLR_COLLECTION + value: {{ .Values.solrInit.collection | quote }} + - name: SOLR_CONFIGSET_NAME + value: {{ .Values.solrInit.configSetName | quote }} + - name: SOLR_NUM_SHARDS + value: {{ .Values.solrInit.numShards | toString | quote }} + - name: SOLR_REPLICATION_FACTOR + value: {{ .Values.solrInit.replicationFactor | toString | quote }} + - name: SOLR_CONF_DIR + value: "/solr-conf" + {{- if and (not .Values.solrInit.existingSecret) .Values.solrInit.adminUser }} + - name: SOLR_ADMIN_USER + value: {{ .Values.solrInit.adminUser | quote }} + - name: SOLR_ADMIN_PASSWORD + value: {{ .Values.solrInit.adminPassword | quote }} + {{- end }} + volumeMounts: + - name: solr-init-script + mountPath: /scripts + readOnly: true + - name: solr-dataverse-conf + mountPath: /solr-conf + readOnly: true + {{- with .Values.solrInit.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.container.port }} + protocol: TCP + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or .Values.extraEnvVars .Values.awsS3.enabled }} + env: + {{- if .Values.extraEnvVars }} + {{- toYaml .Values.extraEnvVars | nindent 12 }} + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: aws_endpoint_url + value: {{ .Values.awsS3.endpointUrl | quote }} + - name: aws_bucket_name + value: {{ .Values.awsS3.bucketName | quote }} + - name: aws_s3_profile + value: {{ .Values.awsS3.profile | quote }} + - name: aws_s3_region + value: {{ .Values.awsS3.region | quote }} + # Force correct SigV4 region (overrides a bad `region` line in the mounted config Secret). + - name: AWS_REGION + value: {{ .Values.awsS3.region | quote }} + - name: AWS_DEFAULT_REGION + value: {{ .Values.awsS3.region | quote }} + # Non-root Payara user cannot write /root/.aws; SDK reads these paths (mounted Secret). + - name: AWS_SHARED_CREDENTIALS_FILE + value: /secrets/aws-cli/.aws/credentials + - name: AWS_CONFIG_FILE + value: /secrets/aws-cli/.aws/config + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumeMounts }} + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- if .Values.docrootPersistence.enabled }} + - name: docroot + mountPath: {{ .Values.docrootPersistence.mountPath }} + {{- end }} + {{- if or .Values.configMap.enabled .Values.mail.enabled }} + - name: init-d + mountPath: {{ .Values.configMap.mountPath }} + readOnly: true + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: init-d + mountPath: /opt/payara/scripts/init.d/006-s3-aws-storage.sh + subPath: 006-s3-aws-storage.sh + readOnly: true + - name: aws-cli + mountPath: /secrets/aws-cli/.aws + readOnly: true + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumes .Values.solrInit.enabled }} + volumes: + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-data{{ end }} + {{- end }} + {{- if .Values.docrootPersistence.enabled }} + - name: docroot + persistentVolumeClaim: + claimName: {{ if .Values.docrootPersistence.existingClaim }}{{ .Values.docrootPersistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-docroot{{ end }} + {{- end }} + {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled }} + - name: init-d + configMap: + name: {{ include "dataverseup.fullname" . }}-config + defaultMode: 0755 + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: aws-cli + secret: + secretName: {{ required "awsS3.existingSecret is required when awsS3.enabled=true" .Values.awsS3.existingSecret | quote }} + items: + - key: {{ .Values.awsS3.secretKeys.credentials | quote }} + path: credentials + - key: {{ .Values.awsS3.secretKeys.config | quote }} + path: config + {{- end }} + {{- if .Values.solrInit.enabled }} + - name: solr-init-script + configMap: + name: {{ include "dataverseup.fullname" . }}-solr-init-script + defaultMode: 0555 + - name: solr-dataverse-conf + configMap: + name: {{ .Values.solrInit.confConfigMap | quote }} + defaultMode: 0444 + {{- end }} + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/dataverseup/templates/hpa.yaml b/charts/dataverseup/templates/hpa.yaml new file mode 100644 index 0000000..6abba32 --- /dev/null +++ b/charts/dataverseup/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "dataverseup.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/ingress.yaml b/charts/dataverseup/templates/ingress.yaml new file mode 100644 index 0000000..72d1556 --- /dev/null +++ b/charts/dataverseup/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "dataverseup.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-deployment.yaml b/charts/dataverseup/templates/internal-solr-deployment.yaml new file mode 100644 index 0000000..858936b --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-deployment.yaml @@ -0,0 +1,112 @@ +{{- if .Values.internalSolr.enabled }} +{{- if not .Values.solrInit.enabled }}{{ fail "internalSolr.enabled requires solrInit.enabled (initContainer waits for core)" }}{{- end }} +{{- if not .Values.solrInit.confConfigMap }}{{ fail "internalSolr.enabled requires solrInit.confConfigMap" }}{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dataverseup.fullname" . }}-solr + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 8 }} + spec: + # Official solr image runs as uid 8983; PVCs often mount root-owned → solr-precreate cannot write solr.xml. + securityContext: + fsGroup: {{ .Values.internalSolr.podSecurityContext.fsGroup }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: solr-install-configset + image: busybox:1.36 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + command: + - sh + - -ec + - | + mkdir -p /configsets/dv + if [ -f /cm/solr-conf.tgz ]; then + tar -xzf /cm/solr-conf.tgz -C /configsets/dv + else + cp -a /cm/. /configsets/dv/ + fi + chown -R 8983:8983 /configsets + volumeMounts: + - name: solr-configset + mountPath: /configsets + - name: solr-conf-cm + mountPath: /cm + readOnly: true + - name: solr-datadir-perms + image: busybox:1.36 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + command: + - sh + - -ec + - | + mkdir -p /var/solr/data + chown -R {{ .Values.internalSolr.podSecurityContext.fsGroup }}:{{ .Values.internalSolr.podSecurityContext.fsGroup }} /var/solr/data + volumeMounts: + - name: solr-datadir + mountPath: /var/solr/data + containers: + - name: solr + image: {{ .Values.internalSolr.image | quote }} + imagePullPolicy: {{ .Values.internalSolr.imagePullPolicy }} + # precreate-core copies CONFIG_SOURCE as a path; "dv" alone is cwd-relative → cp: cannot stat 'dv/'. + args: ["solr-precreate", "dataverse", "/opt/solr/server/solr/configsets/dv"] + ports: + - containerPort: 8983 + name: solr-http + volumeMounts: + - name: solr-configset + mountPath: /opt/solr/server/solr/configsets + readOnly: true + - name: solr-datadir + mountPath: /var/solr/data + readinessProbe: + httpGet: + path: /solr/dataverse/admin/ping + port: 8983 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /solr/dataverse/admin/ping + port: 8983 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 4 + {{- with .Values.internalSolr.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: solr-configset + emptyDir: {} + - name: solr-conf-cm + configMap: + name: {{ .Values.solrInit.confConfigMap | quote }} + - name: solr-datadir + {{- if .Values.internalSolr.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "dataverseup.fullname" . }}-solr-data + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-pvc.yaml b/charts/dataverseup/templates/internal-solr-pvc.yaml new file mode 100644 index 0000000..26ab34e --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-pvc.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.internalSolr.enabled .Values.internalSolr.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-solr-data + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.internalSolr.persistence.size | quote }} + {{- if .Values.internalSolr.persistence.storageClassName }} + storageClassName: {{ .Values.internalSolr.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-service.yaml b/charts/dataverseup/templates/internal-solr-service.yaml new file mode 100644 index 0000000..67a1a30 --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.internalSolr.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dataverseup.fullname" . }}-solr + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 8983 + targetPort: solr-http + protocol: TCP + name: solr-http + selector: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/pvc-docroot.yaml b/charts/dataverseup/templates/pvc-docroot.yaml new file mode 100644 index 0000000..9219097 --- /dev/null +++ b/charts/dataverseup/templates/pvc-docroot.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.docrootPersistence.enabled (not .Values.docrootPersistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-docroot + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: docroot +spec: + accessModes: + - {{ .Values.docrootPersistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.docrootPersistence.size | quote }} + {{- with .Values.docrootPersistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/pvc.yaml b/charts/dataverseup/templates/pvc.yaml new file mode 100644 index 0000000..4eb4f9d --- /dev/null +++ b/charts/dataverseup/templates/pvc.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-data + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/service.yaml b/charts/dataverseup/templates/service.yaml new file mode 100644 index 0000000..f44b3cf --- /dev/null +++ b/charts/dataverseup/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "dataverseup.primarySelectorLabels" . | nindent 4 }} diff --git a/charts/dataverseup/templates/serviceaccount.yaml b/charts/dataverseup/templates/serviceaccount.yaml new file mode 100644 index 0000000..673f63b --- /dev/null +++ b/charts/dataverseup/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "dataverseup.serviceAccountName" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/dataverseup/templates/solr-init-configmap.yaml b/charts/dataverseup/templates/solr-init-configmap.yaml new file mode 100644 index 0000000..43b9b2b --- /dev/null +++ b/charts/dataverseup/templates/solr-init-configmap.yaml @@ -0,0 +1,140 @@ +{{- if .Values.solrInit.enabled }} +{{- $standalone := or .Values.internalSolr.enabled (eq .Values.solrInit.mode "standalone") }} +{{- if and (not $standalone) (not .Values.solrInit.zkConnect) }}{{ fail "solrInit.zkConnect is required when solrInit.mode is cloud and internalSolr.enabled is false" }}{{- end }} +{{- if not .Values.solrInit.confConfigMap }}{{ fail "solrInit.confConfigMap is required when solrInit.enabled (ConfigMap with Dataverse solr conf files)" }}{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-solr-init-script + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +data: + solr-init.sh: | + #!/bin/bash + set -euo pipefail + MODE="${SOLR_INIT_MODE:-cloud}" + if [[ "${MODE}" == "standalone" ]]; then + COLLECTION="${SOLR_COLLECTION:-dataverse}" + BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase}" + AUTH=() + if [[ -n "${SOLR_ADMIN_USER:-}" && -n "${SOLR_ADMIN_PASSWORD:-}" ]]; then + AUTH=(-u "${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}") + fi + echo "solr-init: standalone — waiting for Solr at ${BASE} (core ${COLLECTION}) ..." + for _ in $(seq 1 120); do + if curl -sfS "${AUTH[@]}" "${BASE}/solr/${COLLECTION}/admin/ping" >/dev/null 2>&1; then + echo "solr-init: core ${COLLECTION} is reachable" + exit 0 + fi + sleep 2 + done + echo "solr-init: timeout waiting for Solr core ping" >&2 + exit 1 + fi + # SolrCloud: upload Dataverse configset to ZK, then create collection if missing. + SOLR_BIN="${SOLR_BIN:-/opt/bitnami/solr/bin/solr}" + ZK="${SOLR_ZK_CONNECT:?set solrInit.zkConnect}" + CONF_DIR="${SOLR_CONF_DIR:-/solr-conf}" + COLLECTION="${SOLR_COLLECTION:-dataverse}" + CONFIGSET="${SOLR_CONFIGSET_NAME:-dataverse_config}" + BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase}" + SHARDS="${SOLR_NUM_SHARDS:-1}" + RF="${SOLR_REPLICATION_FACTOR:-2}" + + AUTH=() + if [[ -n "${SOLR_ADMIN_USER:-}" && -n "${SOLR_ADMIN_PASSWORD:-}" ]]; then + AUTH=(-u "${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}") + fi + + echo "solr-init: waiting for Solr admin at ${BASE} ..." + for _ in $(seq 1 90); do + if curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/info/system" >/dev/null 2>&1; then + echo "solr-init: Solr is up" + break + fi + sleep 2 + done + if ! curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/info/system" >/dev/null 2>&1; then + echo "solr-init: timeout waiting for Solr" >&2 + exit 1 + fi + + # zk upconfig -d does not reliably use the mounted ConfigMap path; stage to /tmp first. + # Do not stage under $SOLR_HOME/server/solr/configsets (not writable when runAsNonRoot uid 1001). + # ConfigMap ships solr-conf.tgz (kubectl --from-file=DIR omits subdirs like lang/). + STAGING="/tmp/dataverse_zk_conf" + rm -rf "${STAGING}" + mkdir -p "${STAGING}" + if [[ -f "${CONF_DIR}/solr-conf.tgz" ]]; then + echo "solr-init: extracting solr-conf.tgz into ${STAGING}" + tar --warning=no-unknown-keyword -xzf "${CONF_DIR}/solr-conf.tgz" -C "${STAGING}" || tar xzf "${CONF_DIR}/solr-conf.tgz" -C "${STAGING}" + else + echo "solr-init: no solr-conf.tgz under ${CONF_DIR} — ConfigMap is still the old flat layout (kubectl --from-file=dir omits lang/)." >&2 + echo "solr-init: listing mount (expect solr-conf.tgz or full flat conf from a ConfigMap per Dataverse Solr guide):" >&2 + ls -la "${CONF_DIR}" >&2 || true + POD_NS="" + if [[ -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + POD_NS=$(tr -d '\n' < /var/run/secrets/kubernetes.io/serviceaccount/namespace) + echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory for your Dataverse version (see docs/HELM.md)." >&2 + else + echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory (namespace in kubectl context)." >&2 + fi + cp -aL "${CONF_DIR}/." "${STAGING}/" + fi + UPLOAD_DIR="" + if [[ -f "${STAGING}/solrconfig.xml" ]]; then + UPLOAD_DIR="${STAGING}" + elif [[ -f "${STAGING}/conf/solrconfig.xml" ]]; then + UPLOAD_DIR="${STAGING}/conf" + else + echo "solr-init: missing solrconfig.xml — ConfigMap must be the full Dataverse Solr conf directory, not schema.xml only." >&2 + echo "solr-init: See https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr and docs/HELM.md" >&2 + echo "solr-init: Example: package conf dir as solr-conf.tgz ConfigMap keys (schema.xml, solrconfig.xml, lang/, …)." >&2 + echo "solr-init: Staged files under ${STAGING}:" >&2 + find "${STAGING}" -type f 2>/dev/null | sort >&2 || ls -laR "${STAGING}" >&2 || true + exit 1 + fi + + if [[ ! -f "${UPLOAD_DIR}/stopwords.txt" ]]; then + echo "solr-init: missing stopwords.txt in staged conf (ConfigMap incomplete)." >&2 + echo "solr-init: Rebuild full conf per Dataverse version; see https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr" >&2 + find "${UPLOAD_DIR}" -type f 2>/dev/null | sort | head -60 >&2 || true + exit 1 + fi + if [[ ! -f "${UPLOAD_DIR}/lang/stopwords_en.txt" ]]; then + echo "solr-init: missing lang/stopwords_en.txt — ConfigMap must be solr-conf.tgz from create-solr-conf-configmap.sh (not kubectl --from-file=dir)." >&2 + exit 1 + fi + + SOLR_HOME="$(cd "$(dirname "$SOLR_BIN")/.." && pwd)" + echo "solr-init: uploading configset ${CONFIGSET} to ZK ${ZK} (from ${UPLOAD_DIR})" + (cd "${SOLR_HOME}" && "$SOLR_BIN" zk upconfig -z "${ZK}" -n "${CONFIGSET}" -d "${UPLOAD_DIR}") + + echo "solr-init: checking collection ${COLLECTION}" + LIST_JSON=$(curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/collections?action=LIST") + if echo "${LIST_JSON}" | grep -q "\"${COLLECTION}\""; then + echo "solr-init: collection ${COLLECTION} already exists" + exit 0 + fi + + echo "solr-init: creating collection ${COLLECTION} (shards=${SHARDS} replicas=${RF})" + CREATE_URL="${BASE}/solr/admin/collections?action=CREATE&name=${COLLECTION}&collection.configName=${CONFIGSET}&numShards=${SHARDS}&replicationFactor=${RF}&wt=json" + CREATE_BODY="${TMPDIR:-/tmp}/solr-init-create-body.json" + HTTP_CODE=$(curl -sS "${AUTH[@]}" -o "${CREATE_BODY}" -w "%{http_code}" "${CREATE_URL}") || true + if [[ "${HTTP_CODE}" != "200" ]]; then + echo "solr-init: collection CREATE failed HTTP ${HTTP_CODE}" >&2 + echo "solr-init: Solr response:" >&2 + cat "${CREATE_BODY}" >&2 2>/dev/null || true + echo "solr-init: If you see unknown configset / ZK mismatch, ensure zkConnect matches Solr (often append /solr chroot for Bitnami)." >&2 + exit 1 + fi + if command -v jq >/dev/null 2>&1; then + STATUS=$(jq -r '.responseHeader.status // empty' "${CREATE_BODY}" 2>/dev/null || true) + if [[ -n "${STATUS}" && "${STATUS}" != "0" ]]; then + echo "solr-init: collection CREATE returned non-zero status in JSON" >&2 + cat "${CREATE_BODY}" >&2 + exit 1 + fi + fi + echo "solr-init: done" +{{- end }} diff --git a/charts/dataverseup/templates/tests/test-connection.yaml b/charts/dataverseup/templates/tests/test-connection.yaml new file mode 100644 index 0000000..3567664 --- /dev/null +++ b/charts/dataverseup/templates/tests/test-connection.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "dataverseup.fullname" . }}-test-connection" + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox:1.36 + command: ["wget"] + args: + - "-qO-" + - "http://{{ include "dataverseup.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}/api/info/version" + restartPolicy: Never diff --git a/charts/dataverseup/values-examples/internal-solr-starter.yaml b/charts/dataverseup/values-examples/internal-solr-starter.yaml new file mode 100644 index 0000000..83d9bee --- /dev/null +++ b/charts/dataverseup/values-examples/internal-solr-starter.yaml @@ -0,0 +1,67 @@ +# Starter overrides — NOT a drop-in install. Copy and replace placeholders: +# your Kubernetes namespace +# helm release name (affects resource names unless fullnameOverride is set) +# Postgres hostname (in-cluster DNS) +# ConfigMap name with full Dataverse Solr conf (or solr-conf.tgz key) +# +# Create a Secret (e.g. dataverse-app-env) with DB password and reference it in extraEnvFrom. +# See docs/HELM.md for the full install order. + +fullnameOverride: dataverse + +replicaCount: 1 + +resources: + requests: + memory: 2Gi + cpu: "500m" + limits: + memory: 4Gi + cpu: "2" + +persistence: + enabled: true + size: 20Gi + +podSecurityContext: + fsGroup: 1000 + +extraEnvFrom: + - secretRef: + name: dataverse-app-env + +extraEnvVars: + - name: INIT_SCRIPTS_FOLDER + value: "/opt/payara/init.d" + - name: DATAVERSE_PID_PROVIDERS + value: "demo" + - name: DATAVERSE_PID_DEFAULT_PROVIDER + value: "demo" + - name: DATAVERSE_PID_DEMO_TYPE + value: "FAKE" + - name: DATAVERSE_PID_DEMO_LABEL + value: "demo" + - name: DATAVERSE_PID_DEMO_AUTHORITY + value: "10.5072" + - name: DATAVERSE_PID_DEMO_SHOULDER + value: "FK2/" + +internalSolr: + enabled: true + persistence: + enabled: true + size: 8Gi + +solrInit: + enabled: true + mode: standalone + confConfigMap: "" + solrHttpBase: "http://dataverse-solr..svc.cluster.local:8983" + zkConnect: "" + +bootstrapJob: + enabled: true + helmHook: true + +ingress: + enabled: false diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml new file mode 100644 index 0000000..fc2386c --- /dev/null +++ b/charts/dataverseup/values.yaml @@ -0,0 +1,269 @@ +# Default values for dataverseup. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# Container listens on 8080 (Payara). Service can expose 80 for Ingress. +container: + port: 8080 + +image: + repository: gdcc/dataverse + pullPolicy: IfNotPresent + # Docker Hub uses -noble-rN tags (see .env VERSION); there is no bare 6.10.1 tag. + # If empty, templates fall back to Chart.AppVersion. + tag: "6.10.1-noble-r0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + # targetPort is the named port on the pod (http → container.port). + +# Load env vars from existing cluster objects (place after extraEnvFrom in pod spec; explicit +# extraEnvVars below override same keys). Prefer Secrets for credentials — create them out-of-band +# (kubectl, External Secrets, Sealed Secrets), not in Helm values, so they are not stored in release history. +extraEnvFrom: [] +# - secretRef: +# name: dataverse-credentials +# - configMapRef: +# name: dataverse-nonsecret-config + +# Pass through to the Deployment; use ops/*-deploy.tmpl.yaml + envsubst for per-env values. +extraEnvVars: [] +# - name: EXAMPLE +# value: "from-template" + +livenessProbe: + httpGet: + path: /api/info/version + port: http + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 6 + +readinessProbe: + httpGet: + path: /api/info/version + port: http + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 6 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# RWO PVC for Payara/Dataverse file storage (e.g. /data). Use 1 replica or a ReadWriteMany +# storage class if you scale out. Set existingClaim to bind a PVC created outside this chart. +persistence: + enabled: false + mountPath: /data + size: 20Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" + +# Optional second PVC for DATAVERSE_FILES_DOCROOT (gdcc image default: /dv/docroot). Same pattern as legacy +# dataverse-k8s "docroot" volume, but correct path for gdcc/dataverse — /logos/*, theme uploads, branding. +# https://guides.dataverse.org/en/latest/container/app-image.html#locations +# With RWO EBS, set podSecurityContext.fsGroup: 1000 so user payara can write the volume. +docrootPersistence: + enabled: false + mountPath: /dv/docroot + size: 6Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" + +# Optional ConfigMap created by this chart; keys become filenames under mountPath (e.g. init scripts). +configMap: + enabled: false + mountPath: /opt/payara/init.d + data: {} + # example: + # data: + # "99-custom.sh": | + # #!/bin/sh + # echo ok + +# Mount compose-compatible mail init script (010-mailrelay-set.sh) under configMap.mountPath. +# Pod env (from extraEnvVars): system_email, mailhost, mailuser, no_reply_email, smtp_password, smtp_port, +# socket_port, smtp_auth, smtp_starttls, smtp_type, smtp_enabled — often mapped from SMTP_* in your env template. +# Payara ADMIN_USER / PASSWORD_FILE come from the gdcc image. If system_email is unset, the script no-ops. +mail: + enabled: false + +# Escape hatch: mount Secrets, extra PVCs, projected volumes, or host paths (compose-style /secrets). +# For nested paths from a Secret, use a projected volume: +# volumes: +# - name: app-secrets +# projected: +# sources: +# - secret: +# name: dataverse-secrets +# items: +# - key: db_password +# path: db/password +# - key: admin_password +# path: admin/password +# volumeMounts: +# - name: app-secrets +# mountPath: /secrets +# readOnly: true +volumes: [] + +volumeMounts: [] + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Optional raw initContainers run before the main container (same idea as Hyrax chart solrPreSetupInitContainer). +solrPreSetupInitContainer: [] + +# Standalone Solr (no ZooKeeper) in the same namespace as this release — official solr image, single core +# "dataverse" via solr-precreate. Uses solrInit.confConfigMap (solr-conf.tgz or flat conf). Turn off your +# external SolrCloud auth/complexity for demos; point solrInit.solrHttpBase at this Service (port 8983). +internalSolr: + enabled: false + image: solr:8.11.2 + imagePullPolicy: IfNotPresent + # Official solr image uses uid/gid 8983; fsGroup + init chown make PVC/emptyDir writable for solr-precreate. + podSecurityContext: + fsGroup: 8983 + resources: {} + persistence: + enabled: false + size: 5Gi + storageClassName: "" + +# SolrCloud bootstrap before Payara starts (same pattern as Hyrax load-solr-config initContainer: +# https://github.com/samvera/hyrax/blob/main/chart/hyrax/templates/deployment.yaml ). +# Uses a Solr image for `solr zk upconfig`, not the Dataverse image (which has no Hyrax shell scripts). +# Provide a ConfigMap whose keys are Solr conf filenames (schema.xml, solrconfig.xml, …) from the Dataverse +# release that matches your Dataverse version — see https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr +solrInit: + enabled: false + image: bitnamilegacy/solr:8.11.2-debian-11-r50 + imagePullPolicy: IfNotPresent + # cloud = zk upconfig + Collections API (needs zkConnect). standalone = wait for core ping only (use with internalSolr.enabled). + mode: cloud + # ZooKeeper connect string Solr uses (include chroot if any), e.g. zk-0.zk-hs:2181,zk-1.zk-hs:2181,zk-2.zk-hs:2181/solr + zkConnect: "" + # Base URL for Solr admin HTTP (no path suffix), e.g. http://solr.solr.svc.cluster.local:8983 + solrHttpBase: "http://solr.solr.svc.cluster.local:8983" + collection: dataverse + configSetName: dataverse_config + numShards: 1 + replicationFactor: 2 + # Name of an existing ConfigMap in the release namespace (keys = files under conf/) + confConfigMap: "" + # Optional Secret with SOLR_ADMIN_USER / SOLR_ADMIN_PASSWORD for Solr basic auth + existingSecret: "" + # If existingSecret is empty, optional inline auth (prefer Secret) + adminUser: "" + adminPassword: "" + resources: {} + +# AWS S3 file storage: Secret mount + env (aws_bucket_name, …) plus bundled 006-s3-aws-storage.sh mounted at +# /opt/payara/scripts/init.d/ (base image entrypoint runs that path before Payara starts). See docs/HELM.md. +awsS3: + enabled: false + # Name of a Secret you create out-of-band (keys = secretKeys below). Required when enabled. + existingSecret: "" + bucketName: "your-bucket-name" + endpointUrl: "https://s3.us-west-2.amazonaws.com" + # AWS SigV4 signing region (e.g. us-west-2). Must match bucket/endpoint; NOT the app name. See docs/HELM.md. + region: "us-west-2" + # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. + profile: default + secretKeys: + credentials: credentials + config: config + +# One-shot gdcc/configbaker: root Dataverse, metadata blocks, dataverseAdmin (dev profile), FAKE DOI, etc. +# See docs/HELM.md for install order and smoke tests. +# - helmHook=true (default): Helm post-install hook only (does not run on helm upgrade). Safe for GitOps. +# - helmHook=false: plain Job (name …-bootstrap-once). For `helm template --show-only … | kubectl apply` only; +# do not set helmHook=false in a values file that you helm upgrade repeatedly (Job spec is immutable). +bootstrapJob: + enabled: false + helmHook: true + image: + repository: gdcc/configbaker + tag: "" + pullPolicy: IfNotPresent + command: + - bootstrap.sh + - dev + # Empty = http://..svc.cluster.local: (matches DATAVERSE_URL on Deployment). + dataverseUrl: "" + timeout: 20m + backoffLimit: 2 + ttlSecondsAfterFinished: 86400 + resources: {} diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..d9b0b3c --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,22 @@ +# Deployment notes (working document) + +Rough notes for standing up Dataverse for Notch8. Extend into a full runbook as you validate each environment. + +## Ticket context (internal) + +- **Target:** Dataverse **v6.10** on **AWS** by **April 7, 2026** — functional demo, not necessarily production-hardened. +- **Deliverable:** Working deployment **and documented process + learnings** (this file, plus **[HELM.md](HELM.md)** for Kubernetes). + +## Docker Compose (local / lab) + +See repository **[README.md](../README.md)** — `docker compose up` after `.env` and `secrets/` from examples. + +## Kubernetes / Helm + +See **[HELM.md](HELM.md)** for chart location (`charts/dataverseup`), Secret layout, Solr ConfigMap expectations, and smoke tests. + +## Learnings log + +| Date | Environment | Note | +|------|-------------|------| +| | | | diff --git a/docs/HELM.md b/docs/HELM.md new file mode 100644 index 0000000..59a3663 --- /dev/null +++ b/docs/HELM.md @@ -0,0 +1,99 @@ +# Helm deployment (DataverseUp chart) + +This document describes how to install the **`dataverseup`** Helm chart from this repository. It is written as **working notes** you can extend into a full runbook after the first successful deploy. + +**Prerequisites:** Helm 3, a Kubernetes cluster, `kubectl` configured, a **PostgreSQL** database reachable from the cluster (in-cluster or managed), and a **StorageClass** for any PVCs you enable. + +**Chart path:** `charts/dataverseup` + +## What the chart deploys + +- **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. +- **Optional bootstrap Job** (`gdcc/configbaker`) — `bootstrap.sh dev` (FAKE DOI, `dataverseAdmin`, etc.). Usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). +- **Optional in-cluster Solr** (`internalSolr`) — single-node Solr with core `dataverse`, plus **`solrInit`** initContainer to wait for Solr / upload config (mode **cloud** or **standalone**). +- **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. + +The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). + +## Install flow (recommended order) + +1. **Create namespace** + `kubectl create namespace ` + +2. **Database** + Provision Postgres and a database/user for Dataverse. Note the service DNS name inside the cluster (e.g. `postgres..svc.cluster.local`). + +3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) + Dataverse needs a **full** Solr configuration directory for its version — not `schema.xml` alone. Build a ConfigMap whose keys are the files under that conf directory (or a single `solr-conf.tgz` as produced by your packaging process). See [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +4. **Application Secret** (example name `dataverse-app-env`) + Prefer `stringData` for passwords. Include at least the variables the GDCC image expects for JDBC and Solr (mirror what you use in Docker Compose `.env`). Typical keys include: + + - `DATAVERSE_DB_HOST`, `DATAVERSE_DB_USER`, `DATAVERSE_DB_PASSWORD`, `DATAVERSE_DB_NAME` + - `POSTGRES_SERVER`, `POSTGRES_PORT`, `POSTGRES_DATABASE`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `PGPASSWORD` + - Solr: `SOLR_LOCATION` or `DATAVERSE_SOLR_HOST` / `DATAVERSE_SOLR_PORT` / `DATAVERSE_SOLR_CORE` (match your Solr deployment) + - Public URL / hostname: `DATAVERSE_URL`, `hostname`, `DATAVERSE_SERVICE_HOST` (used by init scripts and UI) + - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) + +5. **Values file** + Start from `charts/dataverseup/values.yaml` and override with a small file (see `charts/dataverseup/values-examples/internal-solr-starter.yaml` for a commented skeleton). At minimum for a first install: + + - `persistence.enabled: true` (file store) + - `extraEnvFrom` pointing at your Secret + - If using bundled Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.mode: standalone`, `solrInit.confConfigMap`, `solrInit.solrHttpBase` matching the in-chart Solr Service + - `bootstrapJob.enabled: true` for first-time seeding + +6. **Lint and render** + + ```bash + helm lint charts/dataverseup -f your-values.yaml + helm template dataverseup charts/dataverseup -f your-values.yaml > /tmp/manifests.yaml + ``` + +7. **Install** + + ```bash + helm upgrade --install dataverseup charts/dataverseup -n -f your-values.yaml --wait --timeout 45m + ``` + +8. **Smoke tests** + + - `kubectl get pods -n ` + - Bootstrap job logs (if enabled): `kubectl logs -n job/...-bootstrap` + - API: port-forward or Ingress → `GET /api/info/version` should return **200** + - UI login (default bootstrap admin from configbaker **dev** profile — **change** before any shared environment) + +9. **Helm test** (optional) + + ```bash + helm test dataverseup -n + ``` + +## Ingress and TLS + +Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. + +## S3 file storage + +1. Create a Secret in the release namespace with keys matching `awsS3.secretKeys` (default: `credentials`, `config`) — same shape as AWS CLI config files. +2. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `endpointUrl`, `region`, `profile`. + +## Upgrades + +- Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). +- Reconcile Solr conf ConfigMap when Solr schema changes. +- If `bootstrapJob.helmHook` is **true**, the bootstrap Job runs on **post-install only**, not on every upgrade (by design). + +## Learnings log + +Append rows as you go (cluster type, storage class, what broke, what fixed it): + +| Date | Cluster | Note | +|------|---------|------| +| | | | + +## References + +- [Running Dataverse in Docker](https://guides.dataverse.org/en/latest/container/running/index.html) (conceptual parity with container env) +- [Application image](https://guides.dataverse.org/en/latest/container/app-image.html) +- [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr) From 6f11ebb5504649a540fe99cd3ff2586de8c4a86e Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 1 Apr 2026 22:24:10 -0700 Subject: [PATCH 02/31] Helm: remove remaining script name from solr-init error text Made-with: Cursor --- charts/dataverseup/templates/solr-init-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/dataverseup/templates/solr-init-configmap.yaml b/charts/dataverseup/templates/solr-init-configmap.yaml index 43b9b2b..12d8f0c 100644 --- a/charts/dataverseup/templates/solr-init-configmap.yaml +++ b/charts/dataverseup/templates/solr-init-configmap.yaml @@ -102,7 +102,7 @@ data: exit 1 fi if [[ ! -f "${UPLOAD_DIR}/lang/stopwords_en.txt" ]]; then - echo "solr-init: missing lang/stopwords_en.txt — ConfigMap must be solr-conf.tgz from create-solr-conf-configmap.sh (not kubectl --from-file=dir)." >&2 + echo "solr-init: missing lang/stopwords_en.txt — ConfigMap must include lang/ (use a packaged solr-conf.tgz or full conf dir) (not kubectl --from-file=dir)." >&2 exit 1 fi From 528e220690dde957f4beb3a45a24478f31ff58d4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 1 Apr 2026 22:26:43 -0700 Subject: [PATCH 03/31] DRY: chart files/ symlinks to init.d for S3 and mail scripts - Single source of truth under init.d/; Helm follows symlinks and helm package inlines file contents for portable tarballs - Align 010-mailrelay-set.sh header comments for Compose + Helm - Document in charts/dataverseup/README.md and docs/HELM.md Made-with: Cursor --- charts/dataverseup/README.md | 4 ++ .../dataverseup/files/006-s3-aws-storage.sh | 67 +------------------ charts/dataverseup/files/010-mailrelay-set.sh | 34 +--------- docs/HELM.md | 4 ++ 4 files changed, 10 insertions(+), 99 deletions(-) mode change 100644 => 120000 charts/dataverseup/files/006-s3-aws-storage.sh mode change 100644 => 120000 charts/dataverseup/files/010-mailrelay-set.sh diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md index ebd67dd..f011ea0 100644 --- a/charts/dataverseup/README.md +++ b/charts/dataverseup/README.md @@ -20,6 +20,10 @@ helm upgrade --install release-name charts/dataverseup -n your-namespace -f your See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, Secret layout, and smoke tests. +## Payara init scripts (S3, mail) + +`files/006-s3-aws-storage.sh` and `files/010-mailrelay-set.sh` are **symbolic links** to the same scripts in the repository root **`init.d/`** (used by Docker Compose). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. + ## Configuration | Key | Purpose | diff --git a/charts/dataverseup/files/006-s3-aws-storage.sh b/charts/dataverseup/files/006-s3-aws-storage.sh deleted file mode 100644 index beee273..0000000 --- a/charts/dataverseup/files/006-s3-aws-storage.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# AWS S3 file store for Dataverse. -# https://guides.dataverse.org/en/latest/installation/config.html#s3-storage -# -# Sourced by the base image entrypoint from /opt/payara/scripts/init.d/ *before* Payara starts. -# `asadmin create-jvm-options` cannot run here (DAS is not up on :4848). We append -# `create-system-properties` lines to POSTBOOT_COMMANDS_FILE like init_2_configure.sh. -# -# Requires env: aws_bucket_name, aws_endpoint_url, aws_s3_profile, aws_s3_region (chart sets these when awsS3.enabled). -# Kubernetes: chart sets AWS_SHARED_CREDENTIALS_FILE / AWS_CONFIG_FILE (mounted Secret). -# Compose (often root): copy credentials into ~/.aws when those vars are unset. - -if [ -n "${aws_bucket_name:-}" ]; then - if [ -z "${AWS_SHARED_CREDENTIALS_FILE:-}" ] && [ -r /secrets/aws-cli/.aws/credentials ]; then - if mkdir -p /root/.aws 2>/dev/null; then - cp -R /secrets/aws-cli/.aws/. /root/.aws/ - else - _aws_dir="${HOME:-/opt/payara}/.aws" - mkdir -p "$_aws_dir" || { _aws_dir="/opt/payara/.aws" && mkdir -p "$_aws_dir"; } - cp -R /secrets/aws-cli/.aws/. "${_aws_dir}/" - fi - fi - - if [ -z "${POSTBOOT_COMMANDS_FILE:-}" ] || [ ! -w "$POSTBOOT_COMMANDS_FILE" ]; then - echo "006-s3-aws-storage: POSTBOOT_COMMANDS_FILE missing or not writable" >&2 - return 1 - fi - - # Keep dataverse.files.local.* so the "local" driver stays registered for existing DB rows that still - # reference that storage identifier. Set default uploads to S3 via storage-driver-id=S3 only. - # Strip prior S3.* and storage-driver-id lines so pod restarts do not duplicate properties. - _pb_pre=$(mktemp) - _pb_dep=$(mktemp) - trap 'rm -f "${_pb_pre:-}" "${_pb_dep:-}"' EXIT - grep -v -E '^create-system-properties dataverse\.files\.storage-driver-id=' "$POSTBOOT_COMMANDS_FILE" \ - | grep -v -E '^create-system-properties dataverse\.files\.S3\.' \ - | grep -v -E '^deploy ' > "$_pb_pre" || true - grep -E '^deploy ' "$POSTBOOT_COMMANDS_FILE" > "$_pb_dep" || true - _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') - # With custom-endpoint-url set, Dataverse's S3AccessIO uses JVM key custom-endpoint-region for SigV4. - # If unset, upstream defaults to the literal string "dataverse" → S3 400 "region 'dataverse' is wrong". - _s3_reg="${aws_s3_region:-${AWS_REGION:-}}" - if [ -z "${_s3_reg}" ]; then - echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when using custom-endpoint-url" >&2 - return 1 - fi - { - cat "$_pb_pre" - echo "create-system-properties dataverse.files.S3.type=s3" - echo "create-system-properties dataverse.files.S3.label=S3" - echo "create-system-properties dataverse.files.S3.bucket-name=${aws_bucket_name}" - echo "create-system-properties dataverse.files.S3.download-redirect=true" - echo "create-system-properties dataverse.files.S3.url-expiration-minutes=120" - echo "create-system-properties dataverse.files.S3.connection-pool-size=4096" - echo "create-system-properties dataverse.files.storage-driver-id=S3" - echo "create-system-properties dataverse.files.S3.profile=${aws_s3_profile}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" - cat "$_pb_dep" - } > "$POSTBOOT_COMMANDS_FILE" - trap - EXIT - rm -f "$_pb_pre" "$_pb_dep" - # Payara is not listening yet; set via Admin UI/API after first boot if needed. - curl -sfS -m 2 -X PUT "http://127.0.0.1:8080/api/admin/settings/:DownloadMethods" -d "native/http" 2>/dev/null || true -fi diff --git a/charts/dataverseup/files/006-s3-aws-storage.sh b/charts/dataverseup/files/006-s3-aws-storage.sh new file mode 120000 index 0000000..e023f77 --- /dev/null +++ b/charts/dataverseup/files/006-s3-aws-storage.sh @@ -0,0 +1 @@ +../../../init.d/006-s3-aws-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/010-mailrelay-set.sh b/charts/dataverseup/files/010-mailrelay-set.sh deleted file mode 100644 index 1577276..0000000 --- a/charts/dataverseup/files/010-mailrelay-set.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Setup mail relay (compose-compatible; mounted when values.mail.enabled). -# https://guides.dataverse.org/en/latest/developers/troubleshooting.html -# -# smtp_enabled: false|0|no skips this script (GitHub vars SMTP_ENABLED). -# smtp_type=plain implies SMTP AUTH after STARTTLS if smtp_auth is unset (Rails-style). -# smtp_auth / smtp_starttls: true|1|yes when set explicitly. -case "${smtp_enabled}" in - false|0|no|NO|False) exit 0 ;; -esac - -if [ "${system_email}" ]; then - curl -X PUT -d ${system_email} http://localhost:8080/api/admin/settings/:SystemEmail - asadmin --user=${ADMIN_USER} --passwordfile=${PASSWORD_FILE} delete-javamail-resource mail/notifyMailSession - - AUTH_PROP="mail.smtp.auth=false" - case "${smtp_auth}" in - true|1|yes|TRUE|Yes) AUTH_PROP="mail.smtp.auth=true" ;; - esac - if [ "${AUTH_PROP}" = "mail.smtp.auth=false" ]; then - case "${smtp_type}" in - plain|PLAIN) AUTH_PROP="mail.smtp.auth=true" ;; - esac - fi - - PROPS="${AUTH_PROP}:mail.smtp.password=${smtp_password}:mail.smtp.port=${smtp_port}:mail.smtp.socketFactory.port=${socket_port}:mail.smtp.socketFactory.fallback=false" - case "${smtp_starttls}" in - true|1|yes|TRUE|Yes) PROPS="${PROPS}:mail.smtp.starttls.enable=true" ;; - esac - - asadmin --user=${ADMIN_USER} --passwordfile=${PASSWORD_FILE} create-javamail-resource --mailhost ${mailhost} --mailuser ${mailuser} --fromaddress ${no_reply_email} --property "${PROPS}" mail/notifyMailSession -fi diff --git a/charts/dataverseup/files/010-mailrelay-set.sh b/charts/dataverseup/files/010-mailrelay-set.sh new file mode 120000 index 0000000..a708548 --- /dev/null +++ b/charts/dataverseup/files/010-mailrelay-set.sh @@ -0,0 +1 @@ +../../../init.d/010-mailrelay-set.sh \ No newline at end of file diff --git a/docs/HELM.md b/docs/HELM.md index 59a3663..6401de6 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -73,6 +73,10 @@ The chart does **not** install PostgreSQL by default. Supply DB settings with ** Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. +## Payara init scripts (DRY with Compose) + +The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. + ## S3 file storage 1. Create a Secret in the release namespace with keys matching `awsS3.secretKeys` (default: `credentials`, `config`) — same shape as AWS CLI config files. From d3cafc549266cbd8ba7869dd1252555cd545915c Mon Sep 17 00:00:00 2001 From: April Rieger Date: Wed, 1 Apr 2026 22:27:13 -0700 Subject: [PATCH 04/31] init.d: clarify 010-mailrelay header for Compose + Helm (remove stale path) Made-with: Cursor --- init.d/010-mailrelay-set.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/init.d/010-mailrelay-set.sh b/init.d/010-mailrelay-set.sh index 6fe8f0d..087ed9a 100644 --- a/init.d/010-mailrelay-set.sh +++ b/init.d/010-mailrelay-set.sh @@ -1,9 +1,10 @@ #!/bin/bash -# Setup mail relay +# Setup mail relay (Docker Compose init.d + Helm ConfigMap when values.mail.enabled). # https://guides.dataverse.org/en/latest/developers/troubleshooting.html # -# smtp_enabled / smtp_type=plain — same behavior as charts/demo-dataverse/files/010-mailrelay-set.sh +# smtp_enabled: false|0|no skips. smtp_type=plain can imply SMTP AUTH when smtp_auth is unset. +# smtp_auth / smtp_starttls: true|1|yes when set explicitly. case "${smtp_enabled}" in false|0|no|NO|False) exit 0 ;; esac From 6e0e241b82fc81f23e40f78a0d860b606622921d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Thu, 2 Apr 2026 09:52:41 -0700 Subject: [PATCH 05/31] Update documentation and pull unecessary local testing values file for k3d cluster --- README.md | 7 +- bin/helm_deploy | 28 ++++++++ charts/dataverseup/README.md | 2 - .../internal-solr-starter.yaml | 67 ------------------- docs/DEPLOYMENT.md | 2 +- docs/HELM.md | 30 ++++++++- 6 files changed, 61 insertions(+), 75 deletions(-) create mode 100755 bin/helm_deploy delete mode 100644 charts/dataverseup/values-examples/internal-solr-starter.yaml diff --git a/README.md b/README.md index b267f3b..4675772 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,12 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G ```bash helm lint charts/dataverseup -helm upgrade --install dataverseup charts/dataverseup -n -f your-values.yaml +HELM_EXTRA_ARGS="--values ./your-values.yaml --wait" ./bin/helm_deploy ``` -Full steps, Secrets, and Solr ConfigMap expectations: **[docs/HELM.md](docs/HELM.md)**. Example values skeleton: **`charts/dataverseup/values-examples/internal-solr-starter.yaml`**. +Wrapper: **`bin/helm_deploy`** (`--atomic`, `--create-namespace`, default **30m** timeout; extend via `HELM_EXTRA_ARGS`). + +Full install order, Secrets, Solr ConfigMap: **[docs/HELM.md](docs/HELM.md)**. ## Layout @@ -50,6 +52,7 @@ Full steps, Secrets, and Solr ConfigMap expectations: **[docs/HELM.md](docs/HELM |------|---------| | `docker-compose.yml` | Stack: Traefik, Postgres, Solr, MinIO (optional), Dataverse, bootstrap, branding | | `charts/dataverseup/` | **Helm chart** for Kubernetes | +| `bin/helm_deploy` | **`helm upgrade --install`** wrapper for the chart | | `.env.example` | Version pins and env template — copy to `.env` | | `secrets.example/` | Payara/Dataverse secret files template — copy to `secrets/` | | `init.d/` | Payara init scripts (local storage, optional S3/MinIO when env set) | diff --git a/bin/helm_deploy b/bin/helm_deploy new file mode 100755 index 0000000..0837eb5 --- /dev/null +++ b/bin/helm_deploy @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Helm upgrade/install for charts/dataverseup (from repository root). +# Pass extra flags via HELM_EXTRA_ARGS, e.g.: +# HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace +# +# A second --timeout in HELM_EXTRA_ARGS overrides the default below (Helm uses the last value). + +set -e + +if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2 + echo " Run from the dataverseup repository root." >&2 + exit 1 +fi + +release_name="$1" +namespace="$2" + +helm upgrade \ + --install \ + --atomic \ + --timeout 30m0s \ + ${HELM_EXTRA_ARGS:-} \ + --namespace="$namespace" \ + --create-namespace \ + "$release_name" \ + "./charts/dataverseup" diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md index f011ea0..2801864 100644 --- a/charts/dataverseup/README.md +++ b/charts/dataverseup/README.md @@ -34,5 +34,3 @@ See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, | `internalSolr` + `solrInit` | In-cluster Solr; requires **full** Solr conf ConfigMap | | `bootstrapJob` | First-time `configbaker` bootstrap | | `ingress` | HTTP routing to Service port 80 | - -Example skeleton: **`values-examples/internal-solr-starter.yaml`**. diff --git a/charts/dataverseup/values-examples/internal-solr-starter.yaml b/charts/dataverseup/values-examples/internal-solr-starter.yaml deleted file mode 100644 index 83d9bee..0000000 --- a/charts/dataverseup/values-examples/internal-solr-starter.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# Starter overrides — NOT a drop-in install. Copy and replace placeholders: -# your Kubernetes namespace -# helm release name (affects resource names unless fullnameOverride is set) -# Postgres hostname (in-cluster DNS) -# ConfigMap name with full Dataverse Solr conf (or solr-conf.tgz key) -# -# Create a Secret (e.g. dataverse-app-env) with DB password and reference it in extraEnvFrom. -# See docs/HELM.md for the full install order. - -fullnameOverride: dataverse - -replicaCount: 1 - -resources: - requests: - memory: 2Gi - cpu: "500m" - limits: - memory: 4Gi - cpu: "2" - -persistence: - enabled: true - size: 20Gi - -podSecurityContext: - fsGroup: 1000 - -extraEnvFrom: - - secretRef: - name: dataverse-app-env - -extraEnvVars: - - name: INIT_SCRIPTS_FOLDER - value: "/opt/payara/init.d" - - name: DATAVERSE_PID_PROVIDERS - value: "demo" - - name: DATAVERSE_PID_DEFAULT_PROVIDER - value: "demo" - - name: DATAVERSE_PID_DEMO_TYPE - value: "FAKE" - - name: DATAVERSE_PID_DEMO_LABEL - value: "demo" - - name: DATAVERSE_PID_DEMO_AUTHORITY - value: "10.5072" - - name: DATAVERSE_PID_DEMO_SHOULDER - value: "FK2/" - -internalSolr: - enabled: true - persistence: - enabled: true - size: 8Gi - -solrInit: - enabled: true - mode: standalone - confConfigMap: "" - solrHttpBase: "http://dataverse-solr..svc.cluster.local:8983" - zkConnect: "" - -bootstrapJob: - enabled: true - helmHook: true - -ingress: - enabled: false diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d9b0b3c..58f3a39 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -13,7 +13,7 @@ See repository **[README.md](../README.md)** — `docker compose up` after `.env ## Kubernetes / Helm -See **[HELM.md](HELM.md)** for chart location (`charts/dataverseup`), Secret layout, Solr ConfigMap expectations, and smoke tests. +See **[HELM.md](HELM.md)** for chart path, **`bin/helm_deploy`**, Secret layout, Solr ConfigMap, and smoke tests. ## Learnings log diff --git a/docs/HELM.md b/docs/HELM.md index 6401de6..6d802fc 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -6,6 +6,20 @@ This document describes how to install the **`dataverseup`** Helm chart from thi **Chart path:** `charts/dataverseup` +## `bin/helm_deploy` (recommended wrapper) + +From the **repository root**, installs or upgrades the chart with **`--install`**, **`--atomic`**, **`--create-namespace`**, and a default **`--timeout 30m0s`** (Payara first boot is slow). + +```text +./bin/helm_deploy RELEASE_NAME NAMESPACE +``` + +Pass extra Helm flags with **`HELM_EXTRA_ARGS`** (values file, longer timeout, etc.). If you pass a **second `--timeout`** in `HELM_EXTRA_ARGS`, it overrides the default (Helm uses the last value). + +```bash +HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace +``` + ## What the chart deploys - **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. @@ -36,7 +50,7 @@ The chart does **not** install PostgreSQL by default. Supply DB settings with ** - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) 5. **Values file** - Start from `charts/dataverseup/values.yaml` and override with a small file (see `charts/dataverseup/values-examples/internal-solr-starter.yaml` for a commented skeleton). At minimum for a first install: + Start from `charts/dataverseup/values.yaml` and override with a small values file of your own. At minimum for a first install: - `persistence.enabled: true` (file store) - `extraEnvFrom` pointing at your Secret @@ -52,8 +66,16 @@ The chart does **not** install PostgreSQL by default. Supply DB settings with ** 7. **Install** + Using the wrapper (from repo root): + ```bash - helm upgrade --install dataverseup charts/dataverseup -n -f your-values.yaml --wait --timeout 45m + HELM_EXTRA_ARGS="--values ./your-values.yaml --wait" ./bin/helm_deploy + ``` + + Raw Helm (equivalent shape): + + ```bash + helm upgrade --install charts/dataverseup -n -f your-values.yaml --wait --timeout 45m ``` 8. **Smoke tests** @@ -66,13 +88,15 @@ The chart does **not** install PostgreSQL by default. Supply DB settings with ** 9. **Helm test** (optional) ```bash - helm test dataverseup -n + helm test -n ``` ## Ingress and TLS Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. +If you terminate TLS or expose the app on a **non-default host port**, keep **`DATAVERSE_URL`** and related hostname settings aligned with the URL users and the app use. + ## Payara init scripts (DRY with Compose) The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. From def61e8ec02d2876c8a89ff2fc8beb1859900f21 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 00:55:48 -0700 Subject: [PATCH 06/31] Update to add jobs for apply branding and seeding the demo collection --- bin/helm_deploy | 29 +++++ branding.env | 1 + charts/dataverseup/Chart.yaml | 2 +- charts/dataverseup/README.md | 4 +- charts/dataverseup/files/apply-branding.sh | 1 + .../files/branding-navbar-logo.svg | 1 + charts/dataverseup/files/branding.env | 1 + .../files/fixtures-seed/dataset-images.json | 1 + .../files/fixtures-seed/dataset-tabular.json | 1 + .../files/fixtures-seed/demo-collection.json | 1 + .../files/fixtures-seed/files_1x1.png | 1 + .../files/fixtures-seed/files_badge.svg | 1 + .../files/fixtures-seed/files_readme.txt | 1 + .../files/fixtures-seed/files_sample.csv | 1 + .../dataverseup/files/k8s-bootstrap-chain.sh | 1 + charts/dataverseup/files/seed-content.sh | 1 + charts/dataverseup/templates/NOTES.txt | 2 +- charts/dataverseup/templates/_helpers.tpl | 28 +++++ .../templates/bootstrap-chain-configmap.yaml | 32 +++++ .../bootstrap-compose-postupgrade-job.yaml | 115 +++++++++++++++++ .../dataverseup/templates/bootstrap-job.yaml | 80 +++++++++++- .../templates/branding-navbar-configmap.yaml | 12 ++ charts/dataverseup/templates/deployment.yaml | 79 ++++++++++-- charts/dataverseup/templates/pvc-docroot.yaml | 2 +- charts/dataverseup/templates/pvc.yaml | 2 +- .../templates/solr-init-configmap.yaml | 8 +- charts/dataverseup/values.yaml | 72 ++++++++--- docs/HELM.md | 40 ++++-- scripts/k8s-bootstrap-chain.sh | 119 ++++++++++++++++++ 29 files changed, 595 insertions(+), 44 deletions(-) create mode 120000 branding.env create mode 120000 charts/dataverseup/files/apply-branding.sh create mode 120000 charts/dataverseup/files/branding-navbar-logo.svg create mode 120000 charts/dataverseup/files/branding.env create mode 120000 charts/dataverseup/files/fixtures-seed/dataset-images.json create mode 120000 charts/dataverseup/files/fixtures-seed/dataset-tabular.json create mode 120000 charts/dataverseup/files/fixtures-seed/demo-collection.json create mode 120000 charts/dataverseup/files/fixtures-seed/files_1x1.png create mode 120000 charts/dataverseup/files/fixtures-seed/files_badge.svg create mode 120000 charts/dataverseup/files/fixtures-seed/files_readme.txt create mode 120000 charts/dataverseup/files/fixtures-seed/files_sample.csv create mode 120000 charts/dataverseup/files/k8s-bootstrap-chain.sh create mode 120000 charts/dataverseup/files/seed-content.sh create mode 100644 charts/dataverseup/templates/bootstrap-chain-configmap.yaml create mode 100644 charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml create mode 100644 charts/dataverseup/templates/branding-navbar-configmap.yaml create mode 100755 scripts/k8s-bootstrap-chain.sh diff --git a/bin/helm_deploy b/bin/helm_deploy index 0837eb5..2328713 100755 --- a/bin/helm_deploy +++ b/bin/helm_deploy @@ -4,10 +4,39 @@ # Pass extra flags via HELM_EXTRA_ARGS, e.g.: # HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace # +# Optional context guard (e.g. local dev): prefix allowlist checked against kubectl +# current-context — deploy/k3d/install.sh sets HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST=k3d- by default. +# # A second --timeout in HELM_EXTRA_ARGS overrides the default below (Helm uses the last value). set -e +# Optional safety: if HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST is set (colon-separated +# prefixes, e.g. k3d-:kind-), refuse to run unless kubectl current-context matches +# one prefix. Not set by default so CI/prod pipelines are unchanged. +if [ -n "${HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST:-}" ]; then + ctx=$(kubectl config current-context 2>/dev/null || echo "") + if [ -z "$ctx" ]; then + echo "helm_deploy: HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST is set but kubectl has no current context." >&2 + exit 1 + fi + matched=0 + old_ifs=$IFS + IFS=: + for prefix in $HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST; do + case "$ctx" in + "${prefix}"*) matched=1; break ;; + esac + done + IFS=$old_ifs + if [ "$matched" != 1 ]; then + echo "helm_deploy: refusing to run: kubectl context is '${ctx}'." >&2 + echo " Expected context to start with one of (colon-separated): ${HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST}" >&2 + echo " Fix: kubectl config use-context or unset HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST." >&2 + exit 1 + fi +fi + if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2 echo " Run from the dataverseup repository root." >&2 diff --git a/branding.env b/branding.env new file mode 120000 index 0000000..e648920 --- /dev/null +++ b/branding.env @@ -0,0 +1 @@ +branding/branding.env \ No newline at end of file diff --git a/charts/dataverseup/Chart.yaml b/charts/dataverseup/Chart.yaml index a287a57..3253651 100644 --- a/charts/dataverseup/Chart.yaml +++ b/charts/dataverseup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.12 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md index 2801864..077c1cd 100644 --- a/charts/dataverseup/README.md +++ b/charts/dataverseup/README.md @@ -31,6 +31,8 @@ See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, | `image.repository` / `image.tag` | GDCC Dataverse image | | `extraEnvFrom` / `extraEnvVars` | DB, Solr, URL, JVM — **use Secrets for credentials** | | `persistence` | RWO PVC for `/data` | -| `internalSolr` + `solrInit` | In-cluster Solr; requires **full** Solr conf ConfigMap | +| `internalSolr` + `solrInit` | **Dedicated Solr 9** pod with this release (not a shared cluster Solr). Default **`solrInit.mode: standalone`**; empty **`solrInit.solrHttpBase`** → chart uses the in-release Solr Service. Core **`dataverse`** (Compose uses **`collection1`**). SolrCloud (`mode: cloud`) needs ZK + **full** conf or `solr-conf.tgz`. | | `bootstrapJob` | First-time `configbaker` bootstrap | | `ingress` | HTTP routing to Service port 80 | + +Solr alignment with Docker Compose (IQSS **`config/`** files, core naming, `solrInit` overrides) is documented in **[docs/HELM.md](../../docs/HELM.md)**. diff --git a/charts/dataverseup/files/apply-branding.sh b/charts/dataverseup/files/apply-branding.sh new file mode 120000 index 0000000..955c071 --- /dev/null +++ b/charts/dataverseup/files/apply-branding.sh @@ -0,0 +1 @@ +../../../scripts/apply-branding.sh \ No newline at end of file diff --git a/charts/dataverseup/files/branding-navbar-logo.svg b/charts/dataverseup/files/branding-navbar-logo.svg new file mode 120000 index 0000000..800438c --- /dev/null +++ b/charts/dataverseup/files/branding-navbar-logo.svg @@ -0,0 +1 @@ +../../../branding/docroot/logos/navbar/logo.svg \ No newline at end of file diff --git a/charts/dataverseup/files/branding.env b/charts/dataverseup/files/branding.env new file mode 120000 index 0000000..49f62f1 --- /dev/null +++ b/charts/dataverseup/files/branding.env @@ -0,0 +1 @@ +../../../branding.env \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/dataset-images.json b/charts/dataverseup/files/fixtures-seed/dataset-images.json new file mode 120000 index 0000000..a2959e6 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/dataset-images.json @@ -0,0 +1 @@ +../../../../fixtures/seed/dataset-images.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/dataset-tabular.json b/charts/dataverseup/files/fixtures-seed/dataset-tabular.json new file mode 120000 index 0000000..a4feaa4 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/dataset-tabular.json @@ -0,0 +1 @@ +../../../../fixtures/seed/dataset-tabular.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/demo-collection.json b/charts/dataverseup/files/fixtures-seed/demo-collection.json new file mode 120000 index 0000000..dd0f9c3 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/demo-collection.json @@ -0,0 +1 @@ +../../../../fixtures/seed/demo-collection.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_1x1.png b/charts/dataverseup/files/fixtures-seed/files_1x1.png new file mode 120000 index 0000000..cb45e4a --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_1x1.png @@ -0,0 +1 @@ +../../../../fixtures/seed/files/1x1.png \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_badge.svg b/charts/dataverseup/files/fixtures-seed/files_badge.svg new file mode 120000 index 0000000..b63ec20 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_badge.svg @@ -0,0 +1 @@ +../../../../fixtures/seed/files/badge.svg \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_readme.txt b/charts/dataverseup/files/fixtures-seed/files_readme.txt new file mode 120000 index 0000000..c873320 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_readme.txt @@ -0,0 +1 @@ +../../../../fixtures/seed/files/readme.txt \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_sample.csv b/charts/dataverseup/files/fixtures-seed/files_sample.csv new file mode 120000 index 0000000..49e1a9f --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_sample.csv @@ -0,0 +1 @@ +../../../../fixtures/seed/files/sample.csv \ No newline at end of file diff --git a/charts/dataverseup/files/k8s-bootstrap-chain.sh b/charts/dataverseup/files/k8s-bootstrap-chain.sh new file mode 120000 index 0000000..d01f1a7 --- /dev/null +++ b/charts/dataverseup/files/k8s-bootstrap-chain.sh @@ -0,0 +1 @@ +../../../scripts/k8s-bootstrap-chain.sh \ No newline at end of file diff --git a/charts/dataverseup/files/seed-content.sh b/charts/dataverseup/files/seed-content.sh new file mode 120000 index 0000000..17c06cc --- /dev/null +++ b/charts/dataverseup/files/seed-content.sh @@ -0,0 +1 @@ +../../../scripts/seed-content.sh \ No newline at end of file diff --git a/charts/dataverseup/templates/NOTES.txt b/charts/dataverseup/templates/NOTES.txt index 08ec125..b69f582 100644 --- a/charts/dataverseup/templates/NOTES.txt +++ b/charts/dataverseup/templates/NOTES.txt @@ -1,5 +1,5 @@ {{- if .Values.internalSolr.enabled }} -In-cluster standalone Solr (no ZooKeeper): Service {{ include "dataverseup.fullname" . }}-solr port 8983 — from another pod in this namespace use http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 . Ensure solrInit.solrHttpBase and DATAVERSE_SOLR_HOST match that host. +Dedicated Solr with this release (not a shared cluster Solr): Service {{ include "dataverseup.fullname" . }}-solr port 8983 — in-namespace URL http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 . If solrInit.solrHttpBase is unset, the chart uses that URL for the init wait. Set app env (e.g. DATAVERSE_SOLR_HOST / SOLR_LOCATION) to the same host:port and core {{ .Values.solrInit.collection }}. {{- end }} {{- if .Values.docrootPersistence.enabled }} diff --git a/charts/dataverseup/templates/_helpers.tpl b/charts/dataverseup/templates/_helpers.tpl index 08f4b41..61edcda 100644 --- a/charts/dataverseup/templates/_helpers.tpl +++ b/charts/dataverseup/templates/_helpers.tpl @@ -50,6 +50,20 @@ app.kubernetes.io/name: {{ include "dataverseup.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* +Bootstrap / configbaker Job pod labels. Must NOT use selectorLabels alone: Deployment matchLabels are only +name+instance, so Job pods would match and `kubectl logs deploy/` can pick the wrong pod. +*/}} +{{- define "dataverseup.bootstrapPodLabels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-bootstrap +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + {{/* Service / CLI label query for the main Dataverse pods only. Pods also set component=primary; Deployment matchLabels stay name+instance only so upgrades do not hit immutable selector changes. @@ -90,3 +104,17 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Solr admin base URL (no path) for solrInit initContainer: explicit solrInit.solrHttpBase, else in-release Solr +Service when internalSolr is enabled, else a placeholder for external / shared cluster Solr (override required). +*/}} +{{- define "dataverseup.solrHttpBase" -}} +{{- if .Values.solrInit.solrHttpBase -}} +{{- .Values.solrInit.solrHttpBase -}} +{{- else if .Values.internalSolr.enabled -}} +http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 +{{- else -}} +http://solr.solr.svc.cluster.local:8983 +{{- end -}} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml new file mode 100644 index 0000000..4874f01 --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.bootstrapJob.enabled (eq .Values.bootstrapJob.mode "compose") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: bootstrap-chain +binaryData: + files_1x1.png: {{ .Files.Get "files/fixtures-seed/files_1x1.png" | b64enc }} +data: + bootstrap-chain.sh: | +{{ .Files.Get "files/k8s-bootstrap-chain.sh" | trim | nindent 4 }} + apply-branding.sh: | +{{ .Files.Get "files/apply-branding.sh" | trim | nindent 4 }} + seed-content.sh: | +{{ .Files.Get "files/seed-content.sh" | trim | nindent 4 }} + branding.env: | +{{ .Files.Get "files/branding.env" | trim | nindent 4 }} + demo-collection.json: | +{{ .Files.Get "files/fixtures-seed/demo-collection.json" | trim | nindent 4 }} + dataset-images.json: | +{{ .Files.Get "files/fixtures-seed/dataset-images.json" | trim | nindent 4 }} + dataset-tabular.json: | +{{ .Files.Get "files/fixtures-seed/dataset-tabular.json" | trim | nindent 4 }} + files_badge.svg: | +{{ .Files.Get "files/fixtures-seed/files_badge.svg" | trim | nindent 4 }} + files_readme.txt: | +{{ .Files.Get "files/fixtures-seed/files_readme.txt" | trim | nindent 4 }} + files_sample.csv: | +{{ .Files.Get "files/fixtures-seed/files_sample.csv" | trim | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml new file mode 100644 index 0000000..f2e69dd --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml @@ -0,0 +1,115 @@ +{{- if and .Values.bootstrapJob.enabled (eq .Values.bootstrapJob.mode "compose") .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob.enabled .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob.existingSecret }} +{{- $pu := .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob }} +{{- $dvUrl := .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap-branding-seed + namespace: {{ .Release.Namespace }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: configbaker-bootstrap-branding-seed + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "20" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +spec: + backoffLimit: {{ .Values.bootstrapJob.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.bootstrapJob.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "dataverseup.bootstrapPodLabels" . | nindent 8 }} + app.kubernetes.io/component: configbaker-bootstrap-branding-seed + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: configbaker + image: "{{ .Values.bootstrapJob.image.repository }}:{{ .Values.bootstrapJob.image.tag | default .Values.image.tag }}" + imagePullPolicy: {{ .Values.bootstrapJob.image.pullPolicy }} + command: ["/bin/bash", "/bootstrap-chain/bootstrap-chain.sh"] + env: + - name: BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP + value: "1" + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_INTERNAL_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_BOOTSTRAP_SECRETS_DIR + value: "/work" + - name: DATAVERSE_BOOTSTRAP_ENV_FILE + value: "/work/api/bootstrap.env" + - name: BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS + value: {{ .Values.bootstrapJob.compose.waitMaxSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_WAIT_SLEEP + value: {{ .Values.bootstrapJob.compose.waitSleepSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_SEED + value: {{ if .Values.bootstrapJob.compose.seed }}"1"{{ else }}"0"{{ end }} + - name: BRANDING_ENV_PATH + value: "/config/branding.env" + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + - name: DATAVERSE_API_TOKEN + valueFrom: + secretKeyRef: + name: {{ $pu.existingSecret | quote }} + key: {{ $pu.secretKey | quote }} + volumeMounts: + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true + {{- with .Values.bootstrapJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml index e25871b..c4eb117 100644 --- a/charts/dataverseup/templates/bootstrap-job.yaml +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -1,4 +1,5 @@ {{- if .Values.bootstrapJob.enabled }} +{{- $dvUrl := .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) }} apiVersion: batch/v1 kind: Job metadata: @@ -19,7 +20,7 @@ spec: template: metadata: labels: - {{- include "dataverseup.selectorLabels" . | nindent 8 }} + {{- include "dataverseup.bootstrapPodLabels" . | nindent 8 }} app.kubernetes.io/component: configbaker-bootstrap spec: restartPolicy: Never @@ -31,14 +32,89 @@ spec: - name: configbaker image: "{{ .Values.bootstrapJob.image.repository }}:{{ .Values.bootstrapJob.image.tag | default .Values.image.tag }}" imagePullPolicy: {{ .Values.bootstrapJob.image.pullPolicy }} + {{- if eq .Values.bootstrapJob.mode "compose" }} + command: ["/bin/bash", "/bootstrap-chain/bootstrap-chain.sh"] + env: + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_INTERNAL_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_BOOTSTRAP_SECRETS_DIR + value: "/work" + - name: DATAVERSE_BOOTSTRAP_ENV_FILE + value: "/work/api/bootstrap.env" + - name: BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS + value: {{ .Values.bootstrapJob.compose.waitMaxSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_WAIT_SLEEP + value: {{ .Values.bootstrapJob.compose.waitSleepSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_SEED + value: {{ if .Values.bootstrapJob.compose.seed }}"1"{{ else }}"0"{{ end }} + - name: BRANDING_ENV_PATH + value: "/config/branding.env" + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + volumeMounts: + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true + {{- else }} command: {{- toYaml .Values.bootstrapJob.command | nindent 12 }} env: - name: DATAVERSE_URL - value: {{ .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) | quote }} + value: {{ $dvUrl | quote }} - name: TIMEOUT value: {{ .Values.bootstrapJob.timeout | quote }} + {{- end }} {{- with .Values.bootstrapJob.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} + {{- if eq .Values.bootstrapJob.mode "compose" }} + volumes: + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} + {{- end }} {{- end }} diff --git a/charts/dataverseup/templates/branding-navbar-configmap.yaml b/charts/dataverseup/templates/branding-navbar-configmap.yaml new file mode 100644 index 0000000..882d284 --- /dev/null +++ b/charts/dataverseup/templates/branding-navbar-configmap.yaml @@ -0,0 +1,12 @@ +{{- if .Values.brandingNavbarLogos.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-branding-navbar + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: branding-navbar +data: + logo.svg: | +{{ .Files.Get "files/branding-navbar-logo.svg" | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/deployment.yaml b/charts/dataverseup/templates/deployment.yaml index e8d9845..700584a 100644 --- a/charts/dataverseup/templates/deployment.yaml +++ b/charts/dataverseup/templates/deployment.yaml @@ -11,6 +11,10 @@ spec: selector: matchLabels: {{- include "dataverseup.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} template: metadata: {{- with .Values.podAnnotations }} @@ -31,7 +35,7 @@ spec: serviceAccountName: {{ include "dataverseup.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- if or .Values.solrInit.enabled .Values.solrPreSetupInitContainer }} + {{- if or .Values.solrInit.enabled .Values.solrPreSetupInitContainer .Values.brandingNavbarLogos.enabled }} initContainers: {{- with .Values.solrPreSetupInitContainer }} {{- toYaml . | nindent 8 }} @@ -42,9 +46,7 @@ spec: image: "{{ .Values.solrInit.image }}" imagePullPolicy: {{ .Values.solrInit.imagePullPolicy }} securityContext: - runAsUser: 1001 - runAsGroup: 1001 - runAsNonRoot: true + {{- toYaml .Values.solrInit.securityContext | nindent 12 }} command: ["/bin/bash", "/scripts/solr-init.sh"] {{- if .Values.solrInit.existingSecret }} envFrom: @@ -59,7 +61,7 @@ spec: - name: SOLR_ZK_CONNECT value: {{ .Values.solrInit.zkConnect | default "" | quote }} - name: SOLR_HTTP_BASE - value: {{ .Values.solrInit.solrHttpBase | quote }} + value: {{ include "dataverseup.solrHttpBase" . | quote }} - name: SOLR_COLLECTION value: {{ .Values.solrInit.collection | quote }} - name: SOLR_CONFIGSET_NAME @@ -70,6 +72,8 @@ spec: value: {{ .Values.solrInit.replicationFactor | toString | quote }} - name: SOLR_CONF_DIR value: "/solr-conf" + - name: SOLR_BIN + value: {{ .Values.solrInit.solrBin | quote }} {{- if and (not .Values.solrInit.existingSecret) .Values.solrInit.adminUser }} - name: SOLR_ADMIN_USER value: {{ .Values.solrInit.adminUser | quote }} @@ -88,6 +92,25 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} + {{- if .Values.brandingNavbarLogos.enabled }} + - name: branding-docroot-init + image: {{ .Values.brandingNavbarLogos.initImage | quote }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + DOCROOT='{{ ternary .Values.docrootPersistence.mountPath "/dv/docroot" .Values.docrootPersistence.enabled }}' + mkdir -p "${DOCROOT}/logos/navbar" + cp /brand/logo.svg "${DOCROOT}/logos/navbar/logo.svg" + chown -R {{ .Values.brandingNavbarLogos.docrootUid }}:{{ .Values.brandingNavbarLogos.docrootGid }} "${DOCROOT}" + volumeMounts: + - name: {{ if .Values.docrootPersistence.enabled }}docroot{{ else }}branding-writable-docroot{{ end }} + mountPath: {{ ternary .Values.docrootPersistence.mountPath "/dv/docroot" .Values.docrootPersistence.enabled | quote }} + - name: branding-navbar-cm + mountPath: /brand + readOnly: true + {{- end }} {{- end }} containers: - name: {{ .Chart.Name }} @@ -99,6 +122,10 @@ spec: - name: http containerPort: {{ .Values.container.port }} protocol: TCP + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -111,8 +138,24 @@ spec: envFrom: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or .Values.extraEnvVars .Values.awsS3.enabled }} + {{- if or .Values.internalSolr.enabled .Values.extraEnvVars .Values.awsS3.enabled }} env: + {{- if .Values.internalSolr.enabled }} + {{- $solrSvc := printf "%s-solr" (include "dataverseup.fullname" .) }} + # GDCC ct profile defaults to solr:8983 + core collection1; override for in-chart Solr (core from solrInit.collection). + - name: DATAVERSE_SOLR_HOST + value: {{ $solrSvc | quote }} + - name: DATAVERSE_SOLR_PORT + value: "8983" + - name: DATAVERSE_SOLR_CORE + value: {{ .Values.solrInit.collection | quote }} + - name: SOLR_SERVICE_HOST + value: {{ $solrSvc | quote }} + - name: SOLR_SERVICE_PORT + value: "8983" + - name: SOLR_LOCATION + value: {{ printf "%s:8983" $solrSvc | quote }} + {{- end }} {{- if .Values.extraEnvVars }} {{- toYaml .Values.extraEnvVars | nindent 12 }} {{- end }} @@ -139,7 +182,7 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumeMounts }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumeMounts }} volumeMounts: {{- if .Values.persistence.enabled }} - name: data @@ -148,6 +191,9 @@ spec: {{- if .Values.docrootPersistence.enabled }} - name: docroot mountPath: {{ .Values.docrootPersistence.mountPath }} + {{- else if .Values.brandingNavbarLogos.enabled }} + - name: branding-writable-docroot + mountPath: /dv/docroot {{- end }} {{- if or .Values.configMap.enabled .Values.mail.enabled }} - name: init-d @@ -167,17 +213,30 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} - {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumes .Values.solrInit.enabled }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumes .Values.solrInit.enabled }} volumes: {{- if .Values.persistence.enabled }} - name: data persistentVolumeClaim: - claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-data{{ end }} + claimName: {{ if empty (.Values.persistence.existingClaim | default "") }}{{ include "dataverseup.fullname" . }}-data{{ else }}{{ .Values.persistence.existingClaim }}{{ end }} {{- end }} {{- if .Values.docrootPersistence.enabled }} - name: docroot persistentVolumeClaim: - claimName: {{ if .Values.docrootPersistence.existingClaim }}{{ .Values.docrootPersistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-docroot{{ end }} + claimName: {{ if empty (.Values.docrootPersistence.existingClaim | default "") }}{{ include "dataverseup.fullname" . }}-docroot{{ else }}{{ .Values.docrootPersistence.existingClaim }}{{ end }} + {{- end }} + {{- if .Values.brandingNavbarLogos.enabled }} + - name: branding-navbar-cm + configMap: + name: {{ include "dataverseup.fullname" . }}-branding-navbar + defaultMode: 0444 + items: + - key: logo.svg + path: logo.svg + {{- if not .Values.docrootPersistence.enabled }} + - name: branding-writable-docroot + emptyDir: {} + {{- end }} {{- end }} {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled }} - name: init-d diff --git a/charts/dataverseup/templates/pvc-docroot.yaml b/charts/dataverseup/templates/pvc-docroot.yaml index 9219097..cc4295e 100644 --- a/charts/dataverseup/templates/pvc-docroot.yaml +++ b/charts/dataverseup/templates/pvc-docroot.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.docrootPersistence.enabled (not .Values.docrootPersistence.existingClaim) }} +{{- if and .Values.docrootPersistence.enabled (empty (.Values.docrootPersistence.existingClaim | default "")) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/charts/dataverseup/templates/pvc.yaml b/charts/dataverseup/templates/pvc.yaml index 4eb4f9d..8339099 100644 --- a/charts/dataverseup/templates/pvc.yaml +++ b/charts/dataverseup/templates/pvc.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +{{- if and .Values.persistence.enabled (empty (.Values.persistence.existingClaim | default "")) }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/charts/dataverseup/templates/solr-init-configmap.yaml b/charts/dataverseup/templates/solr-init-configmap.yaml index 12d8f0c..a702fb6 100644 --- a/charts/dataverseup/templates/solr-init-configmap.yaml +++ b/charts/dataverseup/templates/solr-init-configmap.yaml @@ -15,7 +15,7 @@ data: MODE="${SOLR_INIT_MODE:-cloud}" if [[ "${MODE}" == "standalone" ]]; then COLLECTION="${SOLR_COLLECTION:-dataverse}" - BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase}" + BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase or enable internalSolr for a default}" AUTH=() if [[ -n "${SOLR_ADMIN_USER:-}" && -n "${SOLR_ADMIN_PASSWORD:-}" ]]; then AUTH=(-u "${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}") @@ -32,7 +32,7 @@ data: exit 1 fi # SolrCloud: upload Dataverse configset to ZK, then create collection if missing. - SOLR_BIN="${SOLR_BIN:-/opt/bitnami/solr/bin/solr}" + SOLR_BIN="${SOLR_BIN:-/opt/solr/bin/solr}" ZK="${SOLR_ZK_CONNECT:?set solrInit.zkConnect}" CONF_DIR="${SOLR_CONF_DIR:-/solr-conf}" COLLECTION="${SOLR_COLLECTION:-dataverse}" @@ -60,7 +60,7 @@ data: fi # zk upconfig -d does not reliably use the mounted ConfigMap path; stage to /tmp first. - # Do not stage under $SOLR_HOME/server/solr/configsets (not writable when runAsNonRoot uid 1001). + # Do not stage under $SOLR_HOME/server/solr/configsets (not writable for the solr non-root user). # ConfigMap ships solr-conf.tgz (kubectl --from-file=DIR omits subdirs like lang/). STAGING="/tmp/dataverse_zk_conf" rm -rf "${STAGING}" @@ -125,7 +125,7 @@ data: echo "solr-init: collection CREATE failed HTTP ${HTTP_CODE}" >&2 echo "solr-init: Solr response:" >&2 cat "${CREATE_BODY}" >&2 2>/dev/null || true - echo "solr-init: If you see unknown configset / ZK mismatch, ensure zkConnect matches Solr (often append /solr chroot for Bitnami)." >&2 + echo "solr-init: If you see unknown configset / ZK mismatch, ensure zkConnect matches Solr (include /solr chroot if your distro uses one)." >&2 exit 1 fi if command -v jq >/dev/null 2>&1; then diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index fc2386c..9e333a3 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -4,6 +4,10 @@ replicaCount: 1 +# Optional Deployment strategy (e.g. for replicaCount=1 on small clusters: maxSurge: 0, maxUnavailable: 1 +# avoids two Payara JVMs during a rolling update). +deploymentStrategy: {} + # Container listens on 8080 (Payara). Service can expose 80 for Ingress. container: port: 8080 @@ -81,6 +85,9 @@ readinessProbe: timeoutSeconds: 5 failureThreshold: 6 +# Optional startupProbe (omit key to disable). While failing, liveness/readiness do not run — avoids SIGTERM +# during slow Payara boot when :8080 still refuses connections. See deploy/k3d/values-k3d.yaml.in. + ingress: enabled: false className: "" @@ -138,6 +145,14 @@ docrootPersistence: storageClassName: "" existingClaim: "" +# Navbar logo from chart (symlink to branding/docroot/logos/navbar/logo.svg). Uses a writable emptyDir on +# /dv/docroot (or copies onto docrootPersistence PVC) — read-only ConfigMap mounts break ConfigCheckService. +brandingNavbarLogos: + enabled: false + initImage: busybox:1.36 + docrootUid: 1000 + docrootGid: 1000 + # Optional ConfigMap created by this chart; keys become filenames under mountPath (e.g. init scripts). configMap: enabled: false @@ -186,12 +201,15 @@ affinity: {} # Optional raw initContainers run before the main container (same idea as Hyrax chart solrPreSetupInitContainer). solrPreSetupInitContainer: [] -# Standalone Solr (no ZooKeeper) in the same namespace as this release — official solr image, single core -# "dataverse" via solr-precreate. Uses solrInit.confConfigMap (solr-conf.tgz or flat conf). Turn off your -# external SolrCloud auth/complexity for demos; point solrInit.solrHttpBase at this Service (port 8983). +# Dedicated Solr for this release only — not a shared cluster Solr service or SolrCloud pool. Official +# image, single core "dataverse" via solr-precreate + solrInit.confConfigMap (solr-conf.tgz or flat keys). +# Pair with solrInit.enabled and solrInit.mode: standalone (default). solrHttpBase can stay empty: the chart +# fills http://-solr..svc.cluster.local:8983 when internalSolr is enabled. +# Pin matches docker-compose.yml (IQSS conf/solr for Solr 9 / Lucene 9.x). On Apple Silicon, prefer amd64 +# nodes or cluster image policy — Compose uses platform linux/amd64 for Solr/Dataverse. internalSolr: enabled: false - image: solr:8.11.2 + image: solr:9.10.1 imagePullPolicy: IfNotPresent # Official solr image uses uid/gid 8983; fsGroup + init chown make PVC/emptyDir writable for solr-precreate. podSecurityContext: @@ -207,16 +225,26 @@ internalSolr: # Uses a Solr image for `solr zk upconfig`, not the Dataverse image (which has no Hyrax shell scripts). # Provide a ConfigMap whose keys are Solr conf filenames (schema.xml, solrconfig.xml, …) from the Dataverse # release that matches your Dataverse version — see https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr +# Image aligns with internalSolr / Compose (Solr 9). standalone mode only needs curl+bash; cloud mode runs +# `solr zk upconfig` — use a Solr major that matches your SolrCloud nodes; override solrBin if you use Bitnami. solrInit: enabled: false - image: bitnamilegacy/solr:8.11.2-debian-11-r50 + image: solr:9.10.1 imagePullPolicy: IfNotPresent - # cloud = zk upconfig + Collections API (needs zkConnect). standalone = wait for core ping only (use with internalSolr.enabled). - mode: cloud - # ZooKeeper connect string Solr uses (include chroot if any), e.g. zk-0.zk-hs:2181,zk-1.zk-hs:2181,zk-2.zk-hs:2181/solr + # CLI path inside solrInit.image (official Solr 9: /opt/solr/bin/solr; Bitnami legacy: /opt/bitnami/solr/bin/solr). + solrBin: /opt/solr/bin/solr + securityContext: + runAsUser: 8983 + runAsGroup: 8983 + runAsNonRoot: true + # standalone = wait for core ping (default; use with internalSolr — new Solr pod with this deploy). + # cloud = zk upconfig + Collections API — only if you point at an existing SolrCloud + ZK (not the in-chart Solr). + mode: standalone + # ZooKeeper connect string (required only when mode: cloud), e.g. zk-0.zk-hs:2181/solr zkConnect: "" - # Base URL for Solr admin HTTP (no path suffix), e.g. http://solr.solr.svc.cluster.local:8983 - solrHttpBase: "http://solr.solr.svc.cluster.local:8983" + # Solr admin base URL (no path). Leave empty when internalSolr.enabled — chart sets the in-release Service URL. + # For a shared cluster Solr or SolrCloud front door, set explicitly, e.g. http://solr.mysolr.svc.cluster.local:8983 + solrHttpBase: "" collection: dataverse configSetName: dataverse_config numShards: 1 @@ -246,13 +274,14 @@ awsS3: credentials: credentials config: config -# One-shot gdcc/configbaker: root Dataverse, metadata blocks, dataverseAdmin (dev profile), FAKE DOI, etc. -# See docs/HELM.md for install order and smoke tests. -# - helmHook=true (default): Helm post-install hook only (does not run on helm upgrade). Safe for GitOps. -# - helmHook=false: plain Job (name …-bootstrap-once). For `helm template --show-only … | kubectl apply` only; -# do not set helmHook=false in a values file that you helm upgrade repeatedly (Job spec is immutable). +# gdcc/configbaker Job (post-install hook by default). +# - mode: oneShot — `bootstrapJob.command` only (e.g. bootstrap.sh dev), same as a minimal container demo. +# - mode: compose — like docker compose: configbaker writes API token, then apply-branding.sh, then seed-content.sh +# (repo scripts + fixtures baked into a ConfigMap). Uses emptyDir for bootstrap.env inside the Job only. +# - helmHook=true: runs once on install only (not upgrade). helmHook=false: plain Job (immutable spec on upgrade). bootstrapJob: enabled: false + mode: oneShot helmHook: true image: repository: gdcc/configbaker @@ -267,3 +296,16 @@ bootstrapJob: backoffLimit: 2 ttlSecondsAfterFinished: 86400 resources: {} + compose: + # Wait for /api/info/version before bootstrap.sh (Payara can be slow after install). + waitMaxSeconds: 900 + waitSleepSeconds: 5 + # Run fixtures/seed after branding (set false for bootstrap+branding only). + seed: true + # post-upgrade hook: branding + seed only (no configbaker). Use when the release was upgraded and the + # post-install hook never ran compose, or you need to re-apply after changing branding/fixtures in the chart. + # Create a Secret first, e.g. kubectl create secret generic dv-admin-api-token -n ns --from-literal=token=UUID + postUpgradeBrandingSeedJob: + enabled: false + existingSecret: "" + secretKey: token diff --git a/docs/HELM.md b/docs/HELM.md index 6d802fc..60c333b 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -20,27 +20,49 @@ Pass extra Helm flags with **`HELM_EXTRA_ARGS`** (values file, longer timeout, e HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace ``` + ## What the chart deploys - **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. -- **Optional bootstrap Job** (`gdcc/configbaker`) — `bootstrap.sh dev` (FAKE DOI, `dataverseAdmin`, etc.). Usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). -- **Optional in-cluster Solr** (`internalSolr`) — single-node Solr with core `dataverse`, plus **`solrInit`** initContainer to wait for Solr / upload config (mode **cloud** or **standalone**). +- **Optional bootstrap Job** (`gdcc/configbaker`) — usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). **`bootstrapJob.mode: oneShot`** runs **`bootstrapJob.command`** only (default: `bootstrap.sh dev` — FAKE DOI, `dataverseAdmin`, etc.). **`bootstrapJob.mode: compose`** mirrors local Docker Compose: wait for the API, run configbaker with a writable token file on `emptyDir`, then **`apply-branding.sh`** and **`seed-content.sh`** (fixtures baked into a ConfigMap). Tune waits with **`bootstrapJob.compose`** and allow a longer **`bootstrapJob.timeout`** when seeding. +- **Optional dedicated Solr** (`internalSolr`) — a **new** Solr Deployment/Service in the **same release and namespace** as Dataverse (not wiring into someone else’s shared “cluster Solr”). Default **`solrInit.mode`** is **`standalone`**: the Dataverse pod waits for that Solr core before starting. Use **`solrInit.mode: cloud`** only when Dataverse talks to **SolrCloud + ZooKeeper** you operate separately. - **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). +### Recommended Solr layout: new instance with this deploy + +Enable **`internalSolr.enabled`**, **`solrInit.enabled`**, keep **`solrInit.mode: standalone`**, and supply **`solrInit.confConfigMap`**. Leave **`solrInit.solrHttpBase` empty** — the chart sets the Solr admin URL to the in-release Service (`http://-solr..svc.cluster.local:8983`). Point your app Secret at that same host/port and core (see table below). You do **not** need an existing Solr installation in the cluster. + +## Docker Compose vs Helm (Solr) + +Local **`docker-compose.yml`** and this chart both target **official Solr 9** (`solr:9.10.1`) and IQSS **`conf/solr`** files vendored under repo **`config/`** (refresh from IQSS `develop` or a release tag as in the root **`README.md`**). + +| | Docker Compose | Helm (`internalSolr` + `solrInit`) | +|---|----------------|-----------------------------------| +| Solr image pin | `solr:9.10.1` | `internalSolr.image` / `solrInit.image` default `solr:9.10.1` | +| Default core name | **`collection1`** (see `scripts/solr-initdb/01-ensure-core.sh`) | **`dataverse`** (`solr-precreate` in `internal-solr-deployment.yaml`) | +| App Solr address | `SOLR_LOCATION=solr:8983` (host:port) | With **`internalSolr.enabled`**, the chart sets **`DATAVERSE_SOLR_HOST`**, **`DATAVERSE_SOLR_PORT`**, **`DATAVERSE_SOLR_CORE`**, **`SOLR_SERVICE_*`**, and **`SOLR_LOCATION`** to the in-release Solr Service and **`solrInit.collection`** (default **`dataverse`**). The GDCC `ct` profile otherwise defaults to host **`solr`** and core **`collection1`**, which breaks Kubernetes installs if unset. | + +Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core after precreate. **SolrCloud** (`solrInit.mode: cloud`) still needs a **full** conf tree or **`solr-conf.tgz`** (including `lang/`, `stopwords.txt`, etc.) for `solr zk upconfig` — see [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +### `solrInit` image: standalone (default) vs SolrCloud + +- **Standalone** (default, with **`internalSolr`**): the initContainer **waits** for `/solr//admin/ping` via `curl`; the default **`solr:9.10.1`** image is sufficient. This matches launching a **solo Solr** with the chart instead of consuming a shared cluster Solr Service. +- **Cloud / ZooKeeper** (optional): set **`solrInit.mode: cloud`** and **`solrInit.zkConnect`** when Dataverse uses **SolrCloud** you run elsewhere. The same container runs **`solr zk upconfig`**; use a Solr **major** compatible with that cluster. Override **`solrInit.image`**, **`solrInit.solrBin`**, and **`solrInit.securityContext`** if you use a vendor image (e.g. legacy Bitnami). + ## Install flow (recommended order) -1. **Create namespace** +1. **Create namespace** `kubectl create namespace ` -2. **Database** +2. **Database** Provision Postgres and a database/user for Dataverse. Note the service DNS name inside the cluster (e.g. `postgres..svc.cluster.local`). -3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) +3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) Dataverse needs a **full** Solr configuration directory for its version — not `schema.xml` alone. Build a ConfigMap whose keys are the files under that conf directory (or a single `solr-conf.tgz` as produced by your packaging process). See [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). -4. **Application Secret** (example name `dataverse-app-env`) +4. **Application Secret** (example name `dataverse-app-env`) Prefer `stringData` for passwords. Include at least the variables the GDCC image expects for JDBC and Solr (mirror what you use in Docker Compose `.env`). Typical keys include: - `DATAVERSE_DB_HOST`, `DATAVERSE_DB_USER`, `DATAVERSE_DB_PASSWORD`, `DATAVERSE_DB_NAME` @@ -49,12 +71,12 @@ The chart does **not** install PostgreSQL by default. Supply DB settings with ** - Public URL / hostname: `DATAVERSE_URL`, `hostname`, `DATAVERSE_SERVICE_HOST` (used by init scripts and UI) - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) -5. **Values file** +5. **Values file** Start from `charts/dataverseup/values.yaml` and override with a small values file of your own. At minimum for a first install: - `persistence.enabled: true` (file store) - `extraEnvFrom` pointing at your Secret - - If using bundled Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.mode: standalone`, `solrInit.confConfigMap`, `solrInit.solrHttpBase` matching the in-chart Solr Service + - If using dedicated in-chart Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.confConfigMap`, `solrInit.mode: standalone` (default). Omit `solrInit.solrHttpBase` to use the auto-derived in-release Solr Service URL - `bootstrapJob.enabled: true` for first-time seeding 6. **Lint and render** @@ -110,6 +132,8 @@ The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo ro - Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). - Reconcile Solr conf ConfigMap when Solr schema changes. +- When upgrading **internal Solr** across a **major Solr version** (e.g. 8 → 9), use a **fresh** Solr data volume (new PVC or wipe `internalSolr` persistence) so cores are recreated; same idea as Compose (see root **`README.md`**). +- After bumping **`solrInit`** / **`internalSolr`** images, re-test **SolrCloud** installs (`solr zk` + collection create) in a non-production cluster if you use `solrInit.mode: cloud`. - If `bootstrapJob.helmHook` is **true**, the bootstrap Job runs on **post-install only**, not on every upgrade (by design). ## Learnings log diff --git a/scripts/k8s-bootstrap-chain.sh b/scripts/k8s-bootstrap-chain.sh new file mode 100755 index 0000000..8742baa --- /dev/null +++ b/scripts/k8s-bootstrap-chain.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Helm Job: same order as docker compose dev_bootstrap → dev_branding → dev_seed. +# Runs inside gdcc/configbaker (bootstrap.sh) then applies branding + seed via curl scripts. +set -euo pipefail + +# Helm mounts our chain scripts here; must NOT use /scripts — that shadows gdcc/configbaker's /scripts/bootstrap.sh. +CHAIN_SCRIPTS="${BOOTSTRAP_CHAIN_SCRIPT_DIR:-/bootstrap-chain}" +CONFIGBAKER_BOOTSTRAP="${CONFIGBAKER_BOOTSTRAP_SH:-/scripts/bootstrap.sh}" + +SECRETS_DIR="${DATAVERSE_BOOTSTRAP_SECRETS_DIR:-/secrets}" +TOKEN_FILE="${DATAVERSE_BOOTSTRAP_ENV_FILE:-${SECRETS_DIR}/api/bootstrap.env}" +WAIT_MAX="${BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS:-900}" +SLEEP="${BOOTSTRAP_CHAIN_WAIT_SLEEP:-5}" + +DATAVERSE_INTERNAL_URL="${DATAVERSE_INTERNAL_URL:-}" +if [[ -z "${DATAVERSE_INTERNAL_URL}" ]]; then + echo "k8s-bootstrap-chain: DATAVERSE_INTERNAL_URL is required" >&2 + exit 1 +fi + +try_version() { + curl -sf --max-time 15 "${DATAVERSE_INTERNAL_URL%/}/api/info/version" >/dev/null 2>&1 +} + +echo "k8s-bootstrap-chain: waiting for Dataverse at ${DATAVERSE_INTERNAL_URL} (max ${WAIT_MAX}s) ..." >&2 +elapsed=0 +while [[ "${elapsed}" -lt "${WAIT_MAX}" ]]; do + if try_version; then + echo "k8s-bootstrap-chain: Dataverse API is up" >&2 + break + fi + sleep "${SLEEP}" + elapsed=$((elapsed + SLEEP)) +done +if ! try_version; then + echo "k8s-bootstrap-chain: timeout — Dataverse not reachable" >&2 + exit 1 +fi + +write_api_key_from_token() { + local tok="$1" + mkdir -p "${SECRETS_DIR}/api" + umask 077 + printf '%s\n' "${tok}" >"${SECRETS_DIR}/api/key.tmp" + mv "${SECRETS_DIR}/api/key.tmp" "${SECRETS_DIR}/api/key" + export DATAVERSE_API_TOKEN="${tok}" +} + +# Skip configbaker (e.g. post-upgrade Helm hook): supply DATAVERSE_API_TOKEN from env / secretRef. +if [[ "${BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP:-}" == 1 ]]; then + if [[ -z "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP=1 requires DATAVERSE_API_TOKEN" >&2 + exit 2 + fi + echo "k8s-bootstrap-chain: skipping bootstrap.sh (using existing admin API token)" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" +else + mkdir -p "$(dirname "${TOKEN_FILE}")" + if [[ -d "${TOKEN_FILE}" ]]; then + echo "k8s-bootstrap-chain: ${TOKEN_FILE} is a directory" >&2 + exit 3 + fi + umask 077 + [[ -f "${TOKEN_FILE}" ]] || : >"${TOKEN_FILE}" + + echo "k8s-bootstrap-chain: running configbaker (${CONFIGBAKER_BOOTSTRAP} -e ${TOKEN_FILE} dev) ..." >&2 + if [[ ! -x "${CONFIGBAKER_BOOTSTRAP}" ]]; then + echo "k8s-bootstrap-chain: ${CONFIGBAKER_BOOTSTRAP} missing or not executable (is /scripts shadowed by a volume mount?)" >&2 + exit 127 + fi + "${CONFIGBAKER_BOOTSTRAP}" -e "${TOKEN_FILE}" dev + + # shellcheck disable=SC1090 + source "${TOKEN_FILE}" + if [[ -z "${API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} after bootstrap" >&2 + exit 2 + fi + write_api_key_from_token "${API_TOKEN}" +fi + +export BRANDING_ENV_PATH="${BRANDING_ENV_PATH:-/config/branding.env}" +if [[ -f "${BRANDING_ENV_PATH}" ]]; then + echo "k8s-bootstrap-chain: apply-branding ..." >&2 + export DATAVERSE_INTERNAL_URL + /bin/sh "${CHAIN_SCRIPTS}/apply-branding.sh" +else + echo "k8s-bootstrap-chain: skip branding (no ${BRANDING_ENV_PATH})" >&2 +fi + +layout_seed_from_flat_mount() { + # ConfigMap keys cannot contain '/'; flat keys may be mounted under /seed-flat. + [[ -d /seed-flat ]] || return 0 + mkdir -p /fixtures/seed/files + for f in demo-collection.json dataset-images.json dataset-tabular.json; do + [[ -f "/seed-flat/${f}" ]] && cp "/seed-flat/${f}" "/fixtures/seed/${f}" + done + for pair in "files_1x1.png:1x1.png" "files_badge.svg:badge.svg" "files_readme.txt:readme.txt" "files_sample.csv:sample.csv"; do + key="${pair%%:*}" + name="${pair##*:}" + [[ -f "/seed-flat/${key}" ]] && cp "/seed-flat/${key}" "/fixtures/seed/files/${name}" + done +} + +if [[ "${BOOTSTRAP_CHAIN_SEED:-1}" == "1" ]]; then + layout_seed_from_flat_mount +fi + +if [[ "${BOOTSTRAP_CHAIN_SEED:-1}" == "1" ]] && [[ -f /fixtures/seed/demo-collection.json ]]; then + echo "k8s-bootstrap-chain: seed-content ..." >&2 + export SEED_FIXTURE="${SEED_FIXTURE:-/fixtures/seed/demo-collection.json}" + export SEED_ROOT="${SEED_ROOT:-/fixtures/seed}" + export SEED_PARENT_ALIAS="${SEED_PARENT_ALIAS:-root}" + /bin/sh "${CHAIN_SCRIPTS}/seed-content.sh" +else + echo "k8s-bootstrap-chain: skip seed (BOOTSTRAP_CHAIN_SEED=${BOOTSTRAP_CHAIN_SEED:-} or no fixtures)" >&2 +fi + +echo "k8s-bootstrap-chain: done" >&2 From b2d6973b1ddd09598b97c846631d9426384a5077 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 00:56:56 -0700 Subject: [PATCH 07/31] add github actions deploy and besties deploy values files from current deploy for fine-tuning later --- .github/workflows/deploy.yaml | 131 +++++++++++++++++ ops/besties-deploy.tmpl.yaml | 257 ++++++++++++++++++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 .github/workflows/deploy.yaml create mode 100644 ops/besties-deploy.tmpl.yaml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..4c2953a --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,131 @@ +# In-repo deploy: checkout → kubeconfig → envsubst → bin/helm_deploy. +# Container image comes from chart defaults + ops/-deploy.yaml (not ghcr.io/). +name: Deploy +run-name: Deploy (${{ github.ref_name }} -> ${{ inputs.environment }}) by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + environment: + description: Deploy target (must match ops/-deploy.tmpl.yaml and a GitHub Environment) + required: true + type: choice + default: besties + options: + - besties + debug_enabled: + description: Open an interactive tmate session on the runner before deploy + required: false + type: boolean + default: false + k8s_release_name: + description: Helm release name (leave blank for -) + required: false + type: string + k8s_namespace: + description: Kubernetes namespace (leave blank for -) + required: false + type: string + run_bootstrap_job: + description: Apply a one-shot configbaker Job (bootstrap.sh dev) — use after empty DB; deletes any prior …-bootstrap-once job first + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + container: dtzar/helm-kubectl:3.9.4 + environment: ${{ inputs.environment }} + env: + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + # Optional mail — secrets + Environment variables (see ops/besties-deploy.tmpl.yaml header). + SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} + NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} + SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} + MAIL_SMTP_PASSWORD: ${{ secrets.MAIL_SMTP_PASSWORD }} + SMTP_ADDRESS: ${{ vars.SMTP_ADDRESS }} + SMTP_USER_NAME: ${{ vars.SMTP_USER_NAME }} + SMTP_PORT: ${{ vars.SMTP_PORT }} + SOCKET_PORT: ${{ vars.SOCKET_PORT }} + SMTP_AUTH: ${{ vars.SMTP_AUTH }} + SMTP_STARTTLS: ${{ vars.SMTP_STARTTLS }} + SMTP_TYPE: ${{ vars.SMTP_TYPE }} + SMTP_ENABLED: ${{ vars.SMTP_ENABLED }} + SMTP_DOMAIN: ${{ vars.SMTP_DOMAIN }} + # Bumped every workflow run so the Deployment pod template changes and Kubernetes rolls pods even when + # image.tag and the rest of values are identical (otherwise `helm upgrade` can "succeed" with no rollout). + GITHUB_RUN_ID: ${{ github.run_id }} + HELM_EXPERIMENTAL_OCI: 1 + HELM_EXTRA_ARGS: --values ops/${{ inputs.environment }}-deploy.yaml + KUBECONFIG: ./kubeconfig.yml + KUBECONFIG_FILE: ${{ secrets.KUBECONFIG_FILE }} + steps: + # Local actions under ./.github/actions/* are not on disk until checkout runs — checkout must be first. + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: github.event_name == 'workflow_dispatch' && inputs.debug_enabled + with: + limit-access-to-actor: true + + - name: Deploy with Helm + run: | + set -e + echo "$KUBECONFIG_FILE" | base64 -d > "$KUBECONFIG" + export SMTP_PORT="${SMTP_PORT:-25}" + export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" + export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" + if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then + export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" + fi + DOLLAR=$ envsubst < "ops/${{ inputs.environment }}-deploy.tmpl.yaml" > "ops/${{ inputs.environment }}-deploy.yaml" + REL='${{ inputs.k8s_release_name }}' + NS='${{ inputs.k8s_namespace }}' + if [ -z "$REL" ]; then REL="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi + if [ -z "$NS" ]; then NS="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi + chmod +x bin/helm_deploy + ./bin/helm_deploy "$REL" "$NS" + echo "=== helm status ===" + helm status "$REL" -n "$NS" + echo "=== rollout (Dataverse deployment) ===" + kubectl -n "$NS" rollout status deployment \ + -l "app.kubernetes.io/instance=${REL},app.kubernetes.io/name=demo-dataverse" \ + --timeout=10m + + - name: One-shot configbaker bootstrap Job + if: github.event_name == 'workflow_dispatch' && inputs.run_bootstrap_job + run: | + set -e + echo "$KUBECONFIG_FILE" | base64 -d > "$KUBECONFIG" + export SMTP_PORT="${SMTP_PORT:-25}" + export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" + export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" + if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then + export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" + fi + DOLLAR=$ envsubst < "ops/${{ inputs.environment }}-deploy.tmpl.yaml" > "ops/${{ inputs.environment }}-deploy.yaml" + REL='${{ inputs.k8s_release_name }}' + NS='${{ inputs.k8s_namespace }}' + if [ -z "$REL" ]; then REL="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi + if [ -z "$NS" ]; then NS="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi + JOB="${REL}-bootstrap-once" + kubectl -n "$NS" delete job "$JOB" --ignore-not-found=true + helm template "$REL" ./charts/demo-dataverse \ + --namespace "$NS" \ + $HELM_EXTRA_ARGS \ + --show-only templates/bootstrap-job.yaml \ + --set bootstrapJob.enabled=true \ + --set bootstrapJob.helmHook=false \ + | kubectl apply -f - + kubectl -n "$NS" wait --for=condition=complete "job/$JOB" --timeout=25m + kubectl -n "$NS" logs "job/$JOB" diff --git a/ops/besties-deploy.tmpl.yaml b/ops/besties-deploy.tmpl.yaml new file mode 100644 index 0000000..c59d896 --- /dev/null +++ b/ops/besties-deploy.tmpl.yaml @@ -0,0 +1,257 @@ +# # Helm values for the "besties" GitHub Environment (staging / shared demo). +# # +# # This file uses envsubst: every $VAR below must be exported (or use ../ops/render-besties-deploy.sh). +# # Required: +# # export DB_PASSWORD='...' # Postgres (DATAVERSE_DB_* / POSTGRES_* / PGPASSWORD) +# # export DOLLAR='$' # Keeps literal $ for any $-prefixed YAML if needed later +# # Optional CI: +# # export GITHUB_RUN_ID='...' # pod rollout nonce (Actions sets this automatically) +# # Mail: non-secret SMTP_* + script envs are literal in extraEnvVars below (SendGrid / besties). +# # Secrets via envsubst: SMTP_PASSWORD, SYSTEM_EMAIL; optional SMTP_AUTH (empty = rely on SMTP_TYPE=plain). +# # Legacy MAIL_SMTP_PASSWORD is merged into SMTP_PASSWORD in the deploy workflow if SMTP_PASSWORD is unset. +# # Then: +# # envsubst < ops/besties-deploy.tmpl.yaml > ops/besties-deploy.yaml +# # +# # Besties uses in-chart **standalone Solr** (no ZooKeeper, no HTTP auth) in the **same namespace** as Dataverse. +# # Service DNS (default release/namespace `demo-dataverse-besties`): `…-solr..svc.cluster.local:8983`. +# # If you change `k8s_release_name` / `k8s_namespace` in Deploy, edit **internalSolr** is automatic; edit +# # **solrInit.solrHttpBase** and **DATAVERSE_SOLR_*** / **SOLR_*** host strings below to match `-solr.`. +# # +# # Before first deploy: ConfigMap **dataverse-besties-solr-conf** — see ops/solr-init-setup.md . + +# awsS3: +# enabled: true +# # Name of a Secret you create out-of-band (keys = secretKeys below). +# existingSecret: "aws-s3-credentials" +# bucketName: "demo-dataverse" +# endpointUrl: "https://s3.us-west-2.amazonaws.com" +# region: us-west-2 +# # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. +# profile: default +# secretKeys: +# credentials: credentials +# config: config + +# replicaCount: 1 + +# # CI sets GITHUB_RUN_ID (see .github/workflows/deploy.yaml) so each deploy changes this annotation and forces a +# # new ReplicaSet. Local renders: export GITHUB_RUN_ID=manual-1 (or any string) when you need the same without CI. +# podAnnotations: +# deploy.github.com/run-id: "${GITHUB_RUN_ID}" + +# # Docker Hub tag (not bare 6.10.1). Keeps deploy values explicit so Helm release history cannot stick to a bad tag. +# image: +# repository: gdcc/dataverse +# tag: "6.10.1-noble-r0" + +# resources: +# limits: +# memory: 4Gi +# cpu: "1000m" +# requests: +# memory: 2Gi +# cpu: 100m + +# # Payara runs as uid 1000; fsGroup lets RWO volumes stay writable for /data and /dv/docroot. +# podSecurityContext: +# fsGroup: 1000 + +# # RWO PVC for Dataverse /data — correct for replicaCount: 1. GDCC K8s guide: use ReadWriteMany for /data (and +# # docroot) only for multi-instance Payara; with S3 (besties awsS3) local /data is partly temp/ingest. See +# # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#application-server +# persistence: +# enabled: true +# mountPath: /data +# size: 20Gi +# accessMode: ReadWriteOnce +# storageClassName: "" + +# # Branding / docroot (legacy dataverse-k8s pattern: separate "docroot" PVC). gdcc/dataverse serves /logos/* from +# # DATAVERSE_FILES_DOCROOT → mount at /dv/docroot. Chart creates -docroot or set existingClaim. +# docrootPersistence: +# enabled: true +# mountPath: /dv/docroot +# size: 6Gi +# accessMode: ReadWriteOnce +# storageClassName: "" +# existingClaim: "" + +# ingress: +# enabled: true +# className: nginx-ingress +# annotations: +# nginx.org/client-max-body-size: "0" +# cert-manager.io/cluster-issuer: letsencrypt-production-dns +# hosts: +# - host: demo-dataverse.notch8.cloud +# paths: +# - path: / +# pathType: Prefix +# - host: "*.demo-dataverse.notch8.cloud" +# paths: +# - path: / +# pathType: Prefix +# tls: +# - hosts: +# - demo-dataverse.notch8.cloud +# - "*.demo-dataverse.notch8.cloud" +# secretName: demo-dataverse-tls + +# # Standalone Solr in this release (official solr:8.11.2, core "dataverse", no ZK / no HTTP basic auth). +# internalSolr: +# enabled: true +# image: solr:8.11.2 +# imagePullPolicy: IfNotPresent +# # RWO matches GDCC: standalone Solr index — "ReadWriteOnce ... sufficient" (not SolrCloud yet). +# # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#index-server +# persistence: +# enabled: true +# size: 5Gi +# storageClassName: "" +# resources: {} + +# # Wait for in-cluster Solr core ping before Payara (standalone). For external SolrCloud, set internalSolr.enabled: false, +# # mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/demo-dataverse/values.yaml. +# solrInit: +# enabled: true +# mode: standalone +# image: bitnamilegacy/solr:8.11.2-debian-11-r50 +# imagePullPolicy: IfNotPresent +# zkConnect: "" +# solrHttpBase: "http://demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" +# collection: dataverse +# configSetName: dataverse_config +# numShards: 1 +# replicationFactor: 1 +# confConfigMap: dataverse-besties-solr-conf +# existingSecret: "" +# adminUser: "" +# adminPassword: "" +# resources: {} + +# # Mount init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. +# mail: +# enabled: true + +# # Payara/Dataverse — external Postgres + Solr. Create DB + role on the server first (quoted names if hyphenated); +# # see ops/kubernetes-deploy-checklist.md — error "database \"demo-dataverse\" does not exist" means DB not created. +# extraEnvVars: +# - name: DATAVERSE_DB_HOST +# value: acid-postgres-cluster-delta.postgres.svc.cluster.local +# - name: DATAVERSE_DB_USER +# value: demo-dataverse +# - name: DATAVERSE_DB_PASSWORD +# value: $DB_PASSWORD +# - name: DATAVERSE_DB_NAME +# value: "demo-dataverse" +# - name: POSTGRES_SERVER +# value: acid-postgres-cluster-delta.postgres.svc.cluster.local +# - name: POSTGRES_PORT +# value: "5432" +# - name: POSTGRES_DATABASE +# value: demo-dataverse +# - name: POSTGRES_USER +# value: demo-dataverse +# - name: POSTGRES_PASSWORD +# value: $DB_PASSWORD +# - name: PGPASSWORD +# value: $DB_PASSWORD +# # In-release standalone Solr (same namespace as Dataverse; must match solrInit.solrHttpBase host). +# - name: SOLR_SERVICE_HOST +# value: &solr_host_port "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" +# - name: SOLR_SERVICE_PORT +# value: "8983" +# - name: SOLR_LOCATION +# value: *solr_host_port +# - name: DATAVERSE_SOLR_HOST +# value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" +# - name: DATAVERSE_SOLR_PORT +# value: "8983" +# - name: DATAVERSE_SOLR_CORE +# value: dataverse +# - name: dataverse_solr_host +# value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" +# - name: dataverse_solr_port +# value: "8983" +# - name: dataverse_solr_core +# value: dataverse +# # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style init.d +# # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. +# - name: DATAVERSE_URL +# value: "http://demo-dataverse-besties.demo-dataverse-besties.svc.cluster.local:80" +# # Official image (init_2_configure.sh): dataverse_* env → Payara create-system-properties (dots from underscores). +# # Without these, JSF often cannot resolve the public URL and "/" shows Dataverse "Page Not Found". +# - name: dataverse_siteUrl +# value: https://demo-dataverse.notch8.cloud +# - name: dataverse_fqdn +# value: demo-dataverse.notch8.cloud +# - name: DATAVERSE_SERVICE_HOST +# value: demo-dataverse.notch8.cloud +# - name: hostname +# value: demo-dataverse.notch8.cloud +# - name: JVM_OPTS +# value: "-Xmx2g -Xms2g" +# - name: DATAVERSE_PID_PROVIDERS +# value: demo +# - name: DATAVERSE_PID_DEFAULT_PROVIDER +# value: demo +# - name: DATAVERSE_PID_DEMO_TYPE +# value: FAKE +# - name: DATAVERSE_PID_DEMO_LABEL +# value: demo +# - name: DATAVERSE_PID_DEMO_AUTHORITY +# value: "10.5072" +# - name: DATAVERSE_PID_DEMO_SHOULDER +# value: FK2/ +# - name: INIT_SCRIPTS_FOLDER +# value: /opt/payara/init.d +# # Payara JavaMail + :SystemEmail (init.d/010-mailrelay-set.sh). +# # Literal SMTP_* (override via GitHub Environment if you reintroduce envsubst for these keys). +# - name: SMTP_ADDRESS +# value: "smtp.sendgrid.net" +# - name: SMTP_USER_NAME +# value: "apikey" +# - name: SMTP_TYPE +# value: "plain" +# - name: SMTP_STARTTLS +# value: "true" +# - name: SMTP_PORT +# value: "587" +# - name: SMTP_ENABLED +# value: "true" +# - name: SMTP_DOMAIN +# value: "notch8.com" +# # Script reads these names (aligned with SMTP_* above). +# # no_reply_email → Payara JavaMail --fromaddress (outbound From). No separate Reply-To in the session; +# # mail clients reply to From when Reply-To is absent, so use support@ for both behaviors. +# - name: system_email +# value: "${SYSTEM_EMAIL}" +# - name: mailhost +# value: "smtp.sendgrid.net" +# - name: mailuser +# value: "apikey" +# - name: no_reply_email +# value: "support@notch8.com" +# - name: smtp_password +# value: "${SMTP_PASSWORD}" +# - name: smtp_port +# value: "587" +# - name: socket_port +# value: "587" +# - name: smtp_auth +# value: "${SMTP_AUTH}" +# - name: smtp_starttls +# value: "true" +# - name: smtp_type +# value: "plain" +# - name: smtp_enabled +# value: "true" + +# # gdcc/configbaker after install (post-install hook). Same tag as image.tag above. +# # Does not run on `helm upgrade`. After the DB is bootstrapped once, you may set enabled: false to avoid +# # the hook re-running on a future `helm uninstall` + `helm install` against the same database. +# # If you wiped Postgres later, use GitHub Deploy → "Run configbaker bootstrap job" or apply a Job by hand. +# bootstrapJob: +# enabled: true +# helmHook: true +# timeout: 20m From 1f6c03e7994bc1127916ce51e506b75a352320ed Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 00:57:52 -0700 Subject: [PATCH 08/31] uncomment the values file :face-palm: --- ops/besties-deploy.tmpl.yaml | 486 +++++++++++++++++------------------ 1 file changed, 243 insertions(+), 243 deletions(-) diff --git a/ops/besties-deploy.tmpl.yaml b/ops/besties-deploy.tmpl.yaml index c59d896..0c1d8c4 100644 --- a/ops/besties-deploy.tmpl.yaml +++ b/ops/besties-deploy.tmpl.yaml @@ -1,257 +1,257 @@ -# # Helm values for the "besties" GitHub Environment (staging / shared demo). -# # -# # This file uses envsubst: every $VAR below must be exported (or use ../ops/render-besties-deploy.sh). -# # Required: -# # export DB_PASSWORD='...' # Postgres (DATAVERSE_DB_* / POSTGRES_* / PGPASSWORD) -# # export DOLLAR='$' # Keeps literal $ for any $-prefixed YAML if needed later -# # Optional CI: -# # export GITHUB_RUN_ID='...' # pod rollout nonce (Actions sets this automatically) -# # Mail: non-secret SMTP_* + script envs are literal in extraEnvVars below (SendGrid / besties). -# # Secrets via envsubst: SMTP_PASSWORD, SYSTEM_EMAIL; optional SMTP_AUTH (empty = rely on SMTP_TYPE=plain). -# # Legacy MAIL_SMTP_PASSWORD is merged into SMTP_PASSWORD in the deploy workflow if SMTP_PASSWORD is unset. -# # Then: -# # envsubst < ops/besties-deploy.tmpl.yaml > ops/besties-deploy.yaml -# # -# # Besties uses in-chart **standalone Solr** (no ZooKeeper, no HTTP auth) in the **same namespace** as Dataverse. -# # Service DNS (default release/namespace `demo-dataverse-besties`): `…-solr..svc.cluster.local:8983`. -# # If you change `k8s_release_name` / `k8s_namespace` in Deploy, edit **internalSolr** is automatic; edit -# # **solrInit.solrHttpBase** and **DATAVERSE_SOLR_*** / **SOLR_*** host strings below to match `-solr.`. -# # -# # Before first deploy: ConfigMap **dataverse-besties-solr-conf** — see ops/solr-init-setup.md . +# Helm values for the "besties" GitHub Environment (staging / shared demo). +# +# This file uses envsubst: every $VAR below must be exported (or use ../ops/render-besties-deploy.sh). +# Required: +# export DB_PASSWORD='...' # Postgres (DATAVERSE_DB_* / POSTGRES_* / PGPASSWORD) +# export DOLLAR='$' # Keeps literal $ for any $-prefixed YAML if needed later +# Optional CI: +# export GITHUB_RUN_ID='...' # pod rollout nonce (Actions sets this automatically) +# Mail: non-secret SMTP_* + script envs are literal in extraEnvVars below (SendGrid / besties). +# Secrets via envsubst: SMTP_PASSWORD, SYSTEM_EMAIL; optional SMTP_AUTH (empty = rely on SMTP_TYPE=plain). +# Legacy MAIL_SMTP_PASSWORD is merged into SMTP_PASSWORD in the deploy workflow if SMTP_PASSWORD is unset. +# Then: +# envsubst < ops/besties-deploy.tmpl.yaml > ops/besties-deploy.yaml +# +# Besties uses in-chart **standalone Solr** (no ZooKeeper, no HTTP auth) in the **same namespace** as Dataverse. +# Service DNS (default release/namespace `demo-dataverse-besties`): `…-solr..svc.cluster.local:8983`. +# If you change `k8s_release_name` / `k8s_namespace` in Deploy, edit **internalSolr** is automatic; edit +# **solrInit.solrHttpBase** and **DATAVERSE_SOLR_*** / **SOLR_*** host strings below to match `-solr.`. +# +# Before first deploy: ConfigMap **dataverse-besties-solr-conf** — see ops/solr-init-setup.md . -# awsS3: -# enabled: true -# # Name of a Secret you create out-of-band (keys = secretKeys below). -# existingSecret: "aws-s3-credentials" -# bucketName: "demo-dataverse" -# endpointUrl: "https://s3.us-west-2.amazonaws.com" -# region: us-west-2 -# # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. -# profile: default -# secretKeys: -# credentials: credentials -# config: config +awsS3: + enabled: true + # Name of a Secret you create out-of-band (keys = secretKeys below). + existingSecret: "aws-s3-credentials" + bucketName: "demo-dataverse" + endpointUrl: "https://s3.us-west-2.amazonaws.com" + region: us-west-2 + # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. + profile: default + secretKeys: + credentials: credentials + config: config -# replicaCount: 1 +replicaCount: 1 -# # CI sets GITHUB_RUN_ID (see .github/workflows/deploy.yaml) so each deploy changes this annotation and forces a -# # new ReplicaSet. Local renders: export GITHUB_RUN_ID=manual-1 (or any string) when you need the same without CI. -# podAnnotations: -# deploy.github.com/run-id: "${GITHUB_RUN_ID}" +# CI sets GITHUB_RUN_ID (see .github/workflows/deploy.yaml) so each deploy changes this annotation and forces a +# new ReplicaSet. Local renders: export GITHUB_RUN_ID=manual-1 (or any string) when you need the same without CI. +podAnnotations: + deploy.github.com/run-id: "${GITHUB_RUN_ID}" -# # Docker Hub tag (not bare 6.10.1). Keeps deploy values explicit so Helm release history cannot stick to a bad tag. -# image: -# repository: gdcc/dataverse -# tag: "6.10.1-noble-r0" +# Docker Hub tag (not bare 6.10.1). Keeps deploy values explicit so Helm release history cannot stick to a bad tag. +image: + repository: gdcc/dataverse + tag: "6.10.1-noble-r0" -# resources: -# limits: -# memory: 4Gi -# cpu: "1000m" -# requests: -# memory: 2Gi -# cpu: 100m +resources: + limits: + memory: 4Gi + cpu: "1000m" + requests: + memory: 2Gi + cpu: 100m -# # Payara runs as uid 1000; fsGroup lets RWO volumes stay writable for /data and /dv/docroot. -# podSecurityContext: -# fsGroup: 1000 +# Payara runs as uid 1000; fsGroup lets RWO volumes stay writable for /data and /dv/docroot. +podSecurityContext: + fsGroup: 1000 -# # RWO PVC for Dataverse /data — correct for replicaCount: 1. GDCC K8s guide: use ReadWriteMany for /data (and -# # docroot) only for multi-instance Payara; with S3 (besties awsS3) local /data is partly temp/ingest. See -# # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#application-server -# persistence: -# enabled: true -# mountPath: /data -# size: 20Gi -# accessMode: ReadWriteOnce -# storageClassName: "" +# RWO PVC for Dataverse /data — correct for replicaCount: 1. GDCC K8s guide: use ReadWriteMany for /data (and +# docroot) only for multi-instance Payara; with S3 (besties awsS3) local /data is partly temp/ingest. See +# https://k8s-docs.gdcc.io/en/latest/day1/storage.html#application-server +persistence: + enabled: true + mountPath: /data + size: 20Gi + accessMode: ReadWriteOnce + storageClassName: "" -# # Branding / docroot (legacy dataverse-k8s pattern: separate "docroot" PVC). gdcc/dataverse serves /logos/* from -# # DATAVERSE_FILES_DOCROOT → mount at /dv/docroot. Chart creates -docroot or set existingClaim. -# docrootPersistence: -# enabled: true -# mountPath: /dv/docroot -# size: 6Gi -# accessMode: ReadWriteOnce -# storageClassName: "" -# existingClaim: "" +# Branding / docroot (legacy dataverse-k8s pattern: separate "docroot" PVC). gdcc/dataverse serves /logos/* from +# DATAVERSE_FILES_DOCROOT → mount at /dv/docroot. Chart creates -docroot or set existingClaim. +docrootPersistence: + enabled: true + mountPath: /dv/docroot + size: 6Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" -# ingress: -# enabled: true -# className: nginx-ingress -# annotations: -# nginx.org/client-max-body-size: "0" -# cert-manager.io/cluster-issuer: letsencrypt-production-dns -# hosts: -# - host: demo-dataverse.notch8.cloud -# paths: -# - path: / -# pathType: Prefix -# - host: "*.demo-dataverse.notch8.cloud" -# paths: -# - path: / -# pathType: Prefix -# tls: -# - hosts: -# - demo-dataverse.notch8.cloud -# - "*.demo-dataverse.notch8.cloud" -# secretName: demo-dataverse-tls +ingress: + enabled: true + className: nginx-ingress + annotations: + nginx.org/client-max-body-size: "0" + cert-manager.io/cluster-issuer: letsencrypt-production-dns + hosts: + - host: demo-dataverse.notch8.cloud + paths: + - path: / + pathType: Prefix + - host: "*.demo-dataverse.notch8.cloud" + paths: + - path: / + pathType: Prefix + tls: + - hosts: + - demo-dataverse.notch8.cloud + - "*.demo-dataverse.notch8.cloud" + secretName: demo-dataverse-tls -# # Standalone Solr in this release (official solr:8.11.2, core "dataverse", no ZK / no HTTP basic auth). -# internalSolr: -# enabled: true -# image: solr:8.11.2 -# imagePullPolicy: IfNotPresent -# # RWO matches GDCC: standalone Solr index — "ReadWriteOnce ... sufficient" (not SolrCloud yet). -# # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#index-server -# persistence: -# enabled: true -# size: 5Gi -# storageClassName: "" -# resources: {} +# Standalone Solr in this release (official solr:8.11.2, core "dataverse", no ZK / no HTTP basic auth). +internalSolr: + enabled: true + image: solr:8.11.2 + imagePullPolicy: IfNotPresent + # RWO matches GDCC: standalone Solr index — "ReadWriteOnce ... sufficient" (not SolrCloud yet). + # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#index-server + persistence: + enabled: true + size: 5Gi + storageClassName: "" + resources: {} -# # Wait for in-cluster Solr core ping before Payara (standalone). For external SolrCloud, set internalSolr.enabled: false, -# # mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/demo-dataverse/values.yaml. -# solrInit: -# enabled: true -# mode: standalone -# image: bitnamilegacy/solr:8.11.2-debian-11-r50 -# imagePullPolicy: IfNotPresent -# zkConnect: "" -# solrHttpBase: "http://demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" -# collection: dataverse -# configSetName: dataverse_config -# numShards: 1 -# replicationFactor: 1 -# confConfigMap: dataverse-besties-solr-conf -# existingSecret: "" -# adminUser: "" -# adminPassword: "" -# resources: {} +# Wait for in-cluster Solr core ping before Payara (standalone). For external SolrCloud, set internalSolr.enabled: false, +# mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/demo-dataverse/values.yaml. +solrInit: + enabled: true + mode: standalone + image: bitnamilegacy/solr:8.11.2-debian-11-r50 + imagePullPolicy: IfNotPresent + zkConnect: "" + solrHttpBase: "http://demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" + collection: dataverse + configSetName: dataverse_config + numShards: 1 + replicationFactor: 1 + confConfigMap: dataverse-besties-solr-conf + existingSecret: "" + adminUser: "" + adminPassword: "" + resources: {} -# # Mount init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. -# mail: -# enabled: true +# Mount init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. +mail: + enabled: true -# # Payara/Dataverse — external Postgres + Solr. Create DB + role on the server first (quoted names if hyphenated); -# # see ops/kubernetes-deploy-checklist.md — error "database \"demo-dataverse\" does not exist" means DB not created. -# extraEnvVars: -# - name: DATAVERSE_DB_HOST -# value: acid-postgres-cluster-delta.postgres.svc.cluster.local -# - name: DATAVERSE_DB_USER -# value: demo-dataverse -# - name: DATAVERSE_DB_PASSWORD -# value: $DB_PASSWORD -# - name: DATAVERSE_DB_NAME -# value: "demo-dataverse" -# - name: POSTGRES_SERVER -# value: acid-postgres-cluster-delta.postgres.svc.cluster.local -# - name: POSTGRES_PORT -# value: "5432" -# - name: POSTGRES_DATABASE -# value: demo-dataverse -# - name: POSTGRES_USER -# value: demo-dataverse -# - name: POSTGRES_PASSWORD -# value: $DB_PASSWORD -# - name: PGPASSWORD -# value: $DB_PASSWORD -# # In-release standalone Solr (same namespace as Dataverse; must match solrInit.solrHttpBase host). -# - name: SOLR_SERVICE_HOST -# value: &solr_host_port "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" -# - name: SOLR_SERVICE_PORT -# value: "8983" -# - name: SOLR_LOCATION -# value: *solr_host_port -# - name: DATAVERSE_SOLR_HOST -# value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" -# - name: DATAVERSE_SOLR_PORT -# value: "8983" -# - name: DATAVERSE_SOLR_CORE -# value: dataverse -# - name: dataverse_solr_host -# value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" -# - name: dataverse_solr_port -# value: "8983" -# - name: dataverse_solr_core -# value: dataverse -# # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style init.d -# # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. -# - name: DATAVERSE_URL -# value: "http://demo-dataverse-besties.demo-dataverse-besties.svc.cluster.local:80" -# # Official image (init_2_configure.sh): dataverse_* env → Payara create-system-properties (dots from underscores). -# # Without these, JSF often cannot resolve the public URL and "/" shows Dataverse "Page Not Found". -# - name: dataverse_siteUrl -# value: https://demo-dataverse.notch8.cloud -# - name: dataverse_fqdn -# value: demo-dataverse.notch8.cloud -# - name: DATAVERSE_SERVICE_HOST -# value: demo-dataverse.notch8.cloud -# - name: hostname -# value: demo-dataverse.notch8.cloud -# - name: JVM_OPTS -# value: "-Xmx2g -Xms2g" -# - name: DATAVERSE_PID_PROVIDERS -# value: demo -# - name: DATAVERSE_PID_DEFAULT_PROVIDER -# value: demo -# - name: DATAVERSE_PID_DEMO_TYPE -# value: FAKE -# - name: DATAVERSE_PID_DEMO_LABEL -# value: demo -# - name: DATAVERSE_PID_DEMO_AUTHORITY -# value: "10.5072" -# - name: DATAVERSE_PID_DEMO_SHOULDER -# value: FK2/ -# - name: INIT_SCRIPTS_FOLDER -# value: /opt/payara/init.d -# # Payara JavaMail + :SystemEmail (init.d/010-mailrelay-set.sh). -# # Literal SMTP_* (override via GitHub Environment if you reintroduce envsubst for these keys). -# - name: SMTP_ADDRESS -# value: "smtp.sendgrid.net" -# - name: SMTP_USER_NAME -# value: "apikey" -# - name: SMTP_TYPE -# value: "plain" -# - name: SMTP_STARTTLS -# value: "true" -# - name: SMTP_PORT -# value: "587" -# - name: SMTP_ENABLED -# value: "true" -# - name: SMTP_DOMAIN -# value: "notch8.com" -# # Script reads these names (aligned with SMTP_* above). -# # no_reply_email → Payara JavaMail --fromaddress (outbound From). No separate Reply-To in the session; -# # mail clients reply to From when Reply-To is absent, so use support@ for both behaviors. -# - name: system_email -# value: "${SYSTEM_EMAIL}" -# - name: mailhost -# value: "smtp.sendgrid.net" -# - name: mailuser -# value: "apikey" -# - name: no_reply_email -# value: "support@notch8.com" -# - name: smtp_password -# value: "${SMTP_PASSWORD}" -# - name: smtp_port -# value: "587" -# - name: socket_port -# value: "587" -# - name: smtp_auth -# value: "${SMTP_AUTH}" -# - name: smtp_starttls -# value: "true" -# - name: smtp_type -# value: "plain" -# - name: smtp_enabled -# value: "true" +# Payara/Dataverse — external Postgres + Solr. Create DB + role on the server first (quoted names if hyphenated); +# see ops/kubernetes-deploy-checklist.md — error "database \"demo-dataverse\" does not exist" means DB not created. +extraEnvVars: + - name: DATAVERSE_DB_HOST + value: acid-postgres-cluster-delta.postgres.svc.cluster.local + - name: DATAVERSE_DB_USER + value: demo-dataverse + - name: DATAVERSE_DB_PASSWORD + value: $DB_PASSWORD + - name: DATAVERSE_DB_NAME + value: "demo-dataverse" + - name: POSTGRES_SERVER + value: acid-postgres-cluster-delta.postgres.svc.cluster.local + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DATABASE + value: demo-dataverse + - name: POSTGRES_USER + value: demo-dataverse + - name: POSTGRES_PASSWORD + value: $DB_PASSWORD + - name: PGPASSWORD + value: $DB_PASSWORD + # In-release standalone Solr (same namespace as Dataverse; must match solrInit.solrHttpBase host). + - name: SOLR_SERVICE_HOST + value: &solr_host_port "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" + - name: SOLR_SERVICE_PORT + value: "8983" + - name: SOLR_LOCATION + value: *solr_host_port + - name: DATAVERSE_SOLR_HOST + value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" + - name: DATAVERSE_SOLR_PORT + value: "8983" + - name: DATAVERSE_SOLR_CORE + value: dataverse + - name: dataverse_solr_host + value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" + - name: dataverse_solr_port + value: "8983" + - name: dataverse_solr_core + value: dataverse + # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style init.d + # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. + - name: DATAVERSE_URL + value: "http://demo-dataverse-besties.demo-dataverse-besties.svc.cluster.local:80" + # Official image (init_2_configure.sh): dataverse_* env → Payara create-system-properties (dots from underscores). + # Without these, JSF often cannot resolve the public URL and "/" shows Dataverse "Page Not Found". + - name: dataverse_siteUrl + value: https://demo-dataverse.notch8.cloud + - name: dataverse_fqdn + value: demo-dataverse.notch8.cloud + - name: DATAVERSE_SERVICE_HOST + value: demo-dataverse.notch8.cloud + - name: hostname + value: demo-dataverse.notch8.cloud + - name: JVM_OPTS + value: "-Xmx2g -Xms2g" + - name: DATAVERSE_PID_PROVIDERS + value: demo + - name: DATAVERSE_PID_DEFAULT_PROVIDER + value: demo + - name: DATAVERSE_PID_DEMO_TYPE + value: FAKE + - name: DATAVERSE_PID_DEMO_LABEL + value: demo + - name: DATAVERSE_PID_DEMO_AUTHORITY + value: "10.5072" + - name: DATAVERSE_PID_DEMO_SHOULDER + value: FK2/ + - name: INIT_SCRIPTS_FOLDER + value: /opt/payara/init.d + # Payara JavaMail + :SystemEmail (init.d/010-mailrelay-set.sh). + # Literal SMTP_* (override via GitHub Environment if you reintroduce envsubst for these keys). + - name: SMTP_ADDRESS + value: "smtp.sendgrid.net" + - name: SMTP_USER_NAME + value: "apikey" + - name: SMTP_TYPE + value: "plain" + - name: SMTP_STARTTLS + value: "true" + - name: SMTP_PORT + value: "587" + - name: SMTP_ENABLED + value: "true" + - name: SMTP_DOMAIN + value: "notch8.com" + # Script reads these names (aligned with SMTP_* above). + # no_reply_email → Payara JavaMail --fromaddress (outbound From). No separate Reply-To in the session; + # mail clients reply to From when Reply-To is absent, so use support@ for both behaviors. + - name: system_email + value: "${SYSTEM_EMAIL}" + - name: mailhost + value: "smtp.sendgrid.net" + - name: mailuser + value: "apikey" + - name: no_reply_email + value: "support@notch8.com" + - name: smtp_password + value: "${SMTP_PASSWORD}" + - name: smtp_port + value: "587" + - name: socket_port + value: "587" + - name: smtp_auth + value: "${SMTP_AUTH}" + - name: smtp_starttls + value: "true" + - name: smtp_type + value: "plain" + - name: smtp_enabled + value: "true" -# # gdcc/configbaker after install (post-install hook). Same tag as image.tag above. -# # Does not run on `helm upgrade`. After the DB is bootstrapped once, you may set enabled: false to avoid -# # the hook re-running on a future `helm uninstall` + `helm install` against the same database. -# # If you wiped Postgres later, use GitHub Deploy → "Run configbaker bootstrap job" or apply a Job by hand. -# bootstrapJob: -# enabled: true -# helmHook: true -# timeout: 20m +# gdcc/configbaker after install (post-install hook). Same tag as image.tag above. +# Does not run on `helm upgrade`. After the DB is bootstrapped once, you may set enabled: false to avoid +# the hook re-running on a future `helm uninstall` + `helm install` against the same database. +# If you wiped Postgres later, use GitHub Deploy → "Run configbaker bootstrap job" or apply a Job by hand. +bootstrapJob: + enabled: true + helmHook: true + timeout: 20m From ad4698beddc94aa7d4587611a32d8cb3e19817f2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 10:13:23 -0700 Subject: [PATCH 09/31] Add to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e134889..3016ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ data/ /docroot/ *.war .idea/ +test-k3d-deploy \ No newline at end of file From 1bf1ca531d486d721082e1cd020fce975dee6068 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 10:26:24 -0700 Subject: [PATCH 10/31] Remove local guardrails -- not needed in ci deploys --- bin/helm_deploy | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/bin/helm_deploy b/bin/helm_deploy index 2328713..0837eb5 100755 --- a/bin/helm_deploy +++ b/bin/helm_deploy @@ -4,39 +4,10 @@ # Pass extra flags via HELM_EXTRA_ARGS, e.g.: # HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace # -# Optional context guard (e.g. local dev): prefix allowlist checked against kubectl -# current-context — deploy/k3d/install.sh sets HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST=k3d- by default. -# # A second --timeout in HELM_EXTRA_ARGS overrides the default below (Helm uses the last value). set -e -# Optional safety: if HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST is set (colon-separated -# prefixes, e.g. k3d-:kind-), refuse to run unless kubectl current-context matches -# one prefix. Not set by default so CI/prod pipelines are unchanged. -if [ -n "${HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST:-}" ]; then - ctx=$(kubectl config current-context 2>/dev/null || echo "") - if [ -z "$ctx" ]; then - echo "helm_deploy: HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST is set but kubectl has no current context." >&2 - exit 1 - fi - matched=0 - old_ifs=$IFS - IFS=: - for prefix in $HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST; do - case "$ctx" in - "${prefix}"*) matched=1; break ;; - esac - done - IFS=$old_ifs - if [ "$matched" != 1 ]; then - echo "helm_deploy: refusing to run: kubectl context is '${ctx}'." >&2 - echo " Expected context to start with one of (colon-separated): ${HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST}" >&2 - echo " Fix: kubectl config use-context or unset HELM_DEPLOY_KUBE_CONTEXT_ALLOWLIST." >&2 - exit 1 - fi -fi - if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2 echo " Run from the dataverseup repository root." >&2 From 659fcead55ad47a47170c494fecd68b29005cc53 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 10:38:09 -0700 Subject: [PATCH 11/31] Make it friends for demo --- .github/workflows/deploy.yaml | 6 +++--- ops/{besties-deploy.tmpl.yaml => friends-deploy.tmpl.yaml} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename ops/{besties-deploy.tmpl.yaml => friends-deploy.tmpl.yaml} (100%) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 4c2953a..c1d6ebf 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -10,9 +10,9 @@ on: description: Deploy target (must match ops/-deploy.tmpl.yaml and a GitHub Environment) required: true type: choice - default: besties + default: friends options: - - besties + - friends debug_enabled: description: Open an interactive tmate session on the runner before deploy required: false @@ -42,7 +42,7 @@ jobs: environment: ${{ inputs.environment }} env: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - # Optional mail — secrets + Environment variables (see ops/besties-deploy.tmpl.yaml header). + # Optional mail — secrets + Environment variables (see ops/friends-deploy.tmpl.yaml header). SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} diff --git a/ops/besties-deploy.tmpl.yaml b/ops/friends-deploy.tmpl.yaml similarity index 100% rename from ops/besties-deploy.tmpl.yaml rename to ops/friends-deploy.tmpl.yaml From 51982a2b39b6351f218d5beb8958c77d5e9a0740 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 14:24:58 -0700 Subject: [PATCH 12/31] Draw back on the smartness of the dynamic env vars - put into the values files instaed for user config --- .github/workflows/deploy.yaml | 85 ++++++++++-------- bin/helm_deploy | 8 +- docs/DEPLOYMENT.md | 20 +++++ ...deploy.tmpl.yaml => demo-deploy.tmpl.yaml} | 88 ++++++++++--------- 4 files changed, 117 insertions(+), 84 deletions(-) rename ops/{friends-deploy.tmpl.yaml => demo-deploy.tmpl.yaml} (70%) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index c1d6ebf..40e464a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,4 +1,4 @@ -# In-repo deploy: checkout → kubeconfig → envsubst → bin/helm_deploy. +# In-repo deploy: checkout → kubeconfig → envsubst (secrets only) → bin/helm_deploy. # Container image comes from chart defaults + ops/-deploy.yaml (not ghcr.io/). name: Deploy run-name: Deploy (${{ github.ref_name }} -> ${{ inputs.environment }}) by @${{ github.actor }} @@ -10,9 +10,9 @@ on: description: Deploy target (must match ops/-deploy.tmpl.yaml and a GitHub Environment) required: true type: choice - default: friends + default: demo options: - - friends + - demo debug_enabled: description: Open an interactive tmate session on the runner before deploy required: false @@ -27,10 +27,10 @@ on: required: false type: string run_bootstrap_job: - description: Apply a one-shot configbaker Job (bootstrap.sh dev) — use after empty DB; deletes any prior …-bootstrap-once job first + description: Re-run the bootstrap Job (helmHook=false) — same manifest as chart templates/bootstrap-job.yaml; use if the post-install hook was skipped or you reset Postgres. Chart default mode is oneShot (configbaker bootstrap.sh dev). Deletes the prior Job by name first. required: false type: boolean - default: true + default: false permissions: contents: read @@ -42,7 +42,7 @@ jobs: environment: ${{ inputs.environment }} env: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - # Optional mail — secrets + Environment variables (see ops/friends-deploy.tmpl.yaml header). + # Optional mail — secrets + Environment variables (see ops/demo-deploy.tmpl.yaml header). SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} @@ -56,6 +56,13 @@ jobs: SMTP_TYPE: ${{ vars.SMTP_TYPE }} SMTP_ENABLED: ${{ vars.SMTP_ENABLED }} SMTP_DOMAIN: ${{ vars.SMTP_DOMAIN }} + DEPLOY_ENVIRONMENT: ${{ inputs.environment }} + HELM_RELEASE_NAME: ${{ inputs.k8s_release_name || format('{0}-{1}', inputs.environment, github.event.repository.name) }} + HELM_NAMESPACE: ${{ inputs.k8s_namespace || format('{0}-{1}', inputs.environment, github.event.repository.name) }} + HELM_APP_NAME: ${{ vars.HELM_APP_NAME || github.event.repository.name }} + HELM_CHART_PATH: ${{ vars.HELM_CHART_PATH || './charts/dataverseup' }} + DEPLOY_ROLLOUT_TIMEOUT: ${{ vars.DEPLOY_ROLLOUT_TIMEOUT || '10m' }} + DEPLOY_BOOTSTRAP_JOB_TIMEOUT: ${{ vars.DEPLOY_BOOTSTRAP_JOB_TIMEOUT || '25m' }} # Bumped every workflow run so the Deployment pod template changes and Kubernetes rolls pods even when # image.tag and the rest of values are identical (otherwise `helm upgrade` can "succeed" with no rollout). GITHUB_RUN_ID: ${{ github.run_id }} @@ -72,60 +79,62 @@ jobs: submodules: recursive token: ${{ secrets.GITHUB_TOKEN }} + - name: Validate deploy template exists + run: test -f "ops/${DEPLOY_ENVIRONMENT}-deploy.tmpl.yaml" + - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: github.event_name == 'workflow_dispatch' && inputs.debug_enabled with: limit-access-to-actor: true - - name: Deploy with Helm + - name: Prepare kubeconfig and render deploy values run: | set -e - echo "$KUBECONFIG_FILE" | base64 -d > "$KUBECONFIG" + TMPL="ops/${DEPLOY_ENVIRONMENT}-deploy.tmpl.yaml" + OUT="ops/${DEPLOY_ENVIRONMENT}-deploy.yaml" + echo "$KUBECONFIG_FILE" | base64 -d >"$KUBECONFIG" export SMTP_PORT="${SMTP_PORT:-25}" export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" fi - DOLLAR=$ envsubst < "ops/${{ inputs.environment }}-deploy.tmpl.yaml" > "ops/${{ inputs.environment }}-deploy.yaml" - REL='${{ inputs.k8s_release_name }}' - NS='${{ inputs.k8s_namespace }}' - if [ -z "$REL" ]; then REL="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi - if [ -z "$NS" ]; then NS="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi + # Only secrets + rollout id — hosts, Solr DNS, ingress, and bucket are literals in the *.tmpl.yaml file. + ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' + envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT" + + - name: Deploy with Helm + run: | + set -e chmod +x bin/helm_deploy - ./bin/helm_deploy "$REL" "$NS" + ./bin/helm_deploy "$HELM_RELEASE_NAME" "$HELM_NAMESPACE" echo "=== helm status ===" - helm status "$REL" -n "$NS" + helm status "$HELM_RELEASE_NAME" -n "$HELM_NAMESPACE" echo "=== rollout (Dataverse deployment) ===" - kubectl -n "$NS" rollout status deployment \ - -l "app.kubernetes.io/instance=${REL},app.kubernetes.io/name=demo-dataverse" \ - --timeout=10m + kubectl -n "$HELM_NAMESPACE" rollout status deployment \ + -l "app.kubernetes.io/instance=${HELM_RELEASE_NAME},app.kubernetes.io/name=${HELM_APP_NAME}" \ + --timeout="${DEPLOY_ROLLOUT_TIMEOUT}" - - name: One-shot configbaker bootstrap Job + - name: One-shot bootstrap Job (non-hook) if: github.event_name == 'workflow_dispatch' && inputs.run_bootstrap_job run: | set -e - echo "$KUBECONFIG_FILE" | base64 -d > "$KUBECONFIG" - export SMTP_PORT="${SMTP_PORT:-25}" - export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" - export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" - if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then - export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" - fi - DOLLAR=$ envsubst < "ops/${{ inputs.environment }}-deploy.tmpl.yaml" > "ops/${{ inputs.environment }}-deploy.yaml" - REL='${{ inputs.k8s_release_name }}' - NS='${{ inputs.k8s_namespace }}' - if [ -z "$REL" ]; then REL="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi - if [ -z "$NS" ]; then NS="${{ github.event.repository.name }}-${{ inputs.environment }}"; fi - JOB="${REL}-bootstrap-once" - kubectl -n "$NS" delete job "$JOB" --ignore-not-found=true - helm template "$REL" ./charts/demo-dataverse \ - --namespace "$NS" \ + OUT="$(mktemp)" + trap 'rm -f "$OUT"' EXIT + helm template "$HELM_RELEASE_NAME" "$HELM_CHART_PATH" \ + --namespace "$HELM_NAMESPACE" \ $HELM_EXTRA_ARGS \ --show-only templates/bootstrap-job.yaml \ --set bootstrapJob.enabled=true \ --set bootstrapJob.helmHook=false \ - | kubectl apply -f - - kubectl -n "$NS" wait --for=condition=complete "job/$JOB" --timeout=25m - kubectl -n "$NS" logs "job/$JOB" + >"$OUT" + JOB="$(awk '/^kind: Job$/{j=1} j && /^ name: /{print $2; exit}' "$OUT")" + if [ -z "$JOB" ]; then + echo "Could not parse Job metadata.name from helm template ($HELM_CHART_PATH templates/bootstrap-job.yaml)." >&2 + exit 1 + fi + kubectl -n "$HELM_NAMESPACE" delete job "$JOB" --ignore-not-found=true + kubectl apply -f "$OUT" + kubectl -n "$HELM_NAMESPACE" wait --for=condition=complete "job/$JOB" --timeout="${DEPLOY_BOOTSTRAP_JOB_TIMEOUT}" + kubectl -n "$HELM_NAMESPACE" logs "job/$JOB" diff --git a/bin/helm_deploy b/bin/helm_deploy index 0837eb5..ca149ff 100755 --- a/bin/helm_deploy +++ b/bin/helm_deploy @@ -1,6 +1,7 @@ #!/bin/sh # -# Helm upgrade/install for charts/dataverseup (from repository root). +# Helm upgrade/install for the Dataverse chart (from repository root). +# Chart path defaults to ./charts/dataverseup; override with HELM_CHART_PATH. # Pass extra flags via HELM_EXTRA_ARGS, e.g.: # HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace # @@ -10,12 +11,13 @@ set -e if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2 - echo " Run from the dataverseup repository root." >&2 + echo " Run from the repository root (HELM_CHART_PATH defaults to ./charts/dataverseup)." >&2 exit 1 fi release_name="$1" namespace="$2" +chart_path="${HELM_CHART_PATH:-./charts/dataverseup}" helm upgrade \ --install \ @@ -25,4 +27,4 @@ helm upgrade \ --namespace="$namespace" \ --create-namespace \ "$release_name" \ - "./charts/dataverseup" + "$chart_path" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 58f3a39..0979e9e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -15,6 +15,26 @@ See repository **[README.md](../README.md)** — `docker compose up` after `.env See **[HELM.md](HELM.md)** for chart path, **`bin/helm_deploy`**, Secret layout, Solr ConfigMap, and smoke tests. +### GitHub Actions — Deploy workflow + +The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for secrets** (`DB_PASSWORD`, `SYSTEM_EMAIL`, `SMTP_PASSWORD`, `SMTP_AUTH`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). + +**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail secrets (`SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`, `MAIL_SMTP_PASSWORD`). + +**Repository or Environment variables (optional):** + +| Variable | Purpose | Default if unset | +|----------|---------|------------------| +| `DEPLOY_TOOLBOX_IMAGE` | Job `container` image | `dtzar/helm-kubectl:3.9.4` | +| `HELM_CHART_PATH` | Path passed to `helm` / `bin/helm_deploy` | `./charts/dataverseup` | +| `HELM_APP_NAME` | `app.kubernetes.io/name` for `kubectl rollout status` | `github.event.repository.name` | +| `DEPLOY_ROLLOUT_TIMEOUT` | Rollout wait | `10m` | +| `DEPLOY_BOOTSTRAP_JOB_TIMEOUT` | Bootstrap Job wait | `25m` | + +Default Helm **release** and **namespace** are **`-`** (e.g. `demo-dataverseup`). Override with workflow inputs `k8s_release_name` / `k8s_namespace` when needed. + +**Migrating or renaming a release:** Update the literals in **`ops/-deploy.tmpl.yaml`** (ingress hosts, `dataverse_*` / `DATAVERSE_*` / `hostname`, `solrHttpBase`, `SOLR_*`, `DATAVERSE_URL`, `awsS3.bucketName`, DB names, etc.) so they match the new Helm release and namespace; then align Postgres, S3, TLS, and running workloads. + ## Learnings log | Date | Environment | Note | diff --git a/ops/friends-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml similarity index 70% rename from ops/friends-deploy.tmpl.yaml rename to ops/demo-deploy.tmpl.yaml index 0c1d8c4..408cab9 100644 --- a/ops/friends-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -1,29 +1,23 @@ -# Helm values for the "besties" GitHub Environment (staging / shared demo). +# Helm values for the "demo" GitHub Environment (staging / shared demo). # -# This file uses envsubst: every $VAR below must be exported (or use ../ops/render-besties-deploy.sh). -# Required: -# export DB_PASSWORD='...' # Postgres (DATAVERSE_DB_* / POSTGRES_* / PGPASSWORD) -# export DOLLAR='$' # Keeps literal $ for any $-prefixed YAML if needed later -# Optional CI: -# export GITHUB_RUN_ID='...' # pod rollout nonce (Actions sets this automatically) -# Mail: non-secret SMTP_* + script envs are literal in extraEnvVars below (SendGrid / besties). -# Secrets via envsubst: SMTP_PASSWORD, SYSTEM_EMAIL; optional SMTP_AUTH (empty = rely on SMTP_TYPE=plain). -# Legacy MAIL_SMTP_PASSWORD is merged into SMTP_PASSWORD in the deploy workflow if SMTP_PASSWORD is unset. -# Then: -# envsubst < ops/besties-deploy.tmpl.yaml > ops/besties-deploy.yaml +# **Edit literals in this file** for your environment: public hostname (ingress + dataverse_siteUrl), in-cluster +# Solr/Dataverse Service DNS (solrHttpBase + SOLR_* / DATAVERSE_URL), S3 bucket, Postgres identifiers, etc. +# They must stay consistent with your Helm **release name** and **namespace** (e.g. both `demo-dataverseup`). # -# Besties uses in-chart **standalone Solr** (no ZooKeeper, no HTTP auth) in the **same namespace** as Dataverse. -# Service DNS (default release/namespace `demo-dataverse-besties`): `…-solr..svc.cluster.local:8983`. -# If you change `k8s_release_name` / `k8s_namespace` in Deploy, edit **internalSolr** is automatic; edit -# **solrInit.solrHttpBase** and **DATAVERSE_SOLR_*** / **SOLR_*** host strings below to match `-solr.`. +# **envsubst** (CI: see `.github/workflows/deploy.yaml`) only injects secrets and the rollout nonce — not URLs: +# $DB_PASSWORD, $SYSTEM_EMAIL, $SMTP_PASSWORD, $SMTP_AUTH, ${GITHUB_RUN_ID} +# Local render: +# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SYSTEM_EMAIL=... SMTP_PASSWORD=... SMTP_AUTH= +# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml # -# Before first deploy: ConfigMap **dataverse-besties-solr-conf** — see ops/solr-init-setup.md . +# Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), +# same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/HELM.md**). awsS3: enabled: true # Name of a Secret you create out-of-band (keys = secretKeys below). existingSecret: "aws-s3-credentials" - bucketName: "demo-dataverse" + bucketName: "demo-dataverseup" endpointUrl: "https://s3.us-west-2.amazonaws.com" region: us-west-2 # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. @@ -83,25 +77,27 @@ ingress: nginx.org/client-max-body-size: "0" cert-manager.io/cluster-issuer: letsencrypt-production-dns hosts: - - host: demo-dataverse.notch8.cloud + - host: "demo-dataverseup.notch8.cloud" paths: - path: / pathType: Prefix - - host: "*.demo-dataverse.notch8.cloud" + - host: "*.demo-dataverseup.notch8.cloud" paths: - path: / pathType: Prefix tls: - hosts: - - demo-dataverse.notch8.cloud - - "*.demo-dataverse.notch8.cloud" - secretName: demo-dataverse-tls + - "demo-dataverseup.notch8.cloud" + - "*.demo-dataverseup.notch8.cloud" + secretName: "demo-dataverseup-tls" -# Standalone Solr in this release (official solr:8.11.2, core "dataverse", no ZK / no HTTP basic auth). +# Standalone Solr — **solr:9.10.1** matches **docker-compose.yml** and **charts/dataverseup/values.yaml** (IQSS Solr 9 conf). internalSolr: enabled: true - image: solr:8.11.2 + image: solr:9.10.1 imagePullPolicy: IfNotPresent + podSecurityContext: + fsGroup: 8983 # RWO matches GDCC: standalone Solr index — "ReadWriteOnce ... sufficient" (not SolrCloud yet). # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#index-server persistence: @@ -111,19 +107,25 @@ internalSolr: resources: {} # Wait for in-cluster Solr core ping before Payara (standalone). For external SolrCloud, set internalSolr.enabled: false, -# mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/demo-dataverse/values.yaml. +# mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/dataverseup/values.yaml. solrInit: enabled: true mode: standalone - image: bitnamilegacy/solr:8.11.2-debian-11-r50 + image: solr:9.10.1 imagePullPolicy: IfNotPresent + solrBin: /opt/solr/bin/solr + securityContext: + runAsUser: 8983 + runAsGroup: 8983 + runAsNonRoot: true zkConnect: "" - solrHttpBase: "http://demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" + # Must match in-cluster Solr Service DNS for this release/namespace (edit if your Helm release or ns differs). + solrHttpBase: "http://demo-dataverseup-solr.demo-dataverseup.svc.cluster.local:8983" collection: dataverse configSetName: dataverse_config numShards: 1 replicationFactor: 1 - confConfigMap: dataverse-besties-solr-conf + confConfigMap: dataverse-solr-conf existingSecret: "" adminUser: "" adminPassword: "" @@ -134,43 +136,43 @@ mail: enabled: true # Payara/Dataverse — external Postgres + Solr. Create DB + role on the server first (quoted names if hyphenated); -# see ops/kubernetes-deploy-checklist.md — error "database \"demo-dataverse\" does not exist" means DB not created. +# see ops/kubernetes-deploy-checklist.md — DB name here must exist on the server (e.g. demo-dataverseup). extraEnvVars: - name: DATAVERSE_DB_HOST value: acid-postgres-cluster-delta.postgres.svc.cluster.local - name: DATAVERSE_DB_USER - value: demo-dataverse + value: "demo-dataverseup" - name: DATAVERSE_DB_PASSWORD value: $DB_PASSWORD - name: DATAVERSE_DB_NAME - value: "demo-dataverse" + value: "demo-dataverseup" - name: POSTGRES_SERVER value: acid-postgres-cluster-delta.postgres.svc.cluster.local - name: POSTGRES_PORT value: "5432" - name: POSTGRES_DATABASE - value: demo-dataverse + value: "demo-dataverseup" - name: POSTGRES_USER - value: demo-dataverse + value: "demo-dataverseup" - name: POSTGRES_PASSWORD value: $DB_PASSWORD - name: PGPASSWORD value: $DB_PASSWORD - # In-release standalone Solr (same namespace as Dataverse; must match solrInit.solrHttpBase host). + # In-cluster Solr (edit host if release/namespace differ from demo-dataverseup). - name: SOLR_SERVICE_HOST - value: &solr_host_port "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local:8983" + value: &solr_host_port "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local:8983" - name: SOLR_SERVICE_PORT value: "8983" - name: SOLR_LOCATION value: *solr_host_port - name: DATAVERSE_SOLR_HOST - value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" + value: "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local" - name: DATAVERSE_SOLR_PORT value: "8983" - name: DATAVERSE_SOLR_CORE value: dataverse - name: dataverse_solr_host - value: "demo-dataverse-besties-solr.demo-dataverse-besties.svc.cluster.local" + value: "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local" - name: dataverse_solr_port value: "8983" - name: dataverse_solr_core @@ -178,17 +180,17 @@ extraEnvVars: # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style init.d # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. - name: DATAVERSE_URL - value: "http://demo-dataverse-besties.demo-dataverse-besties.svc.cluster.local:80" + value: "http://demo-dataverseup.demo-dataverseup.svc.cluster.local:80" # Official image (init_2_configure.sh): dataverse_* env → Payara create-system-properties (dots from underscores). # Without these, JSF often cannot resolve the public URL and "/" shows Dataverse "Page Not Found". - name: dataverse_siteUrl - value: https://demo-dataverse.notch8.cloud + value: "https://demo-dataverseup.notch8.cloud" - name: dataverse_fqdn - value: demo-dataverse.notch8.cloud + value: "demo-dataverseup" - name: DATAVERSE_SERVICE_HOST - value: demo-dataverse.notch8.cloud + value: "demo-dataverseup" - name: hostname - value: demo-dataverse.notch8.cloud + value: "demo-dataverseup" - name: JVM_OPTS value: "-Xmx2g -Xms2g" - name: DATAVERSE_PID_PROVIDERS From ad734c84e1d36f51209ae2c296914fa9d7ec08ac Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 15:59:17 -0700 Subject: [PATCH 13/31] Update env vars for internal solr --- ops/demo-deploy.tmpl.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 408cab9..1e6c51d 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -158,19 +158,8 @@ extraEnvVars: value: $DB_PASSWORD - name: PGPASSWORD value: $DB_PASSWORD - # In-cluster Solr (edit host if release/namespace differ from demo-dataverseup). - - name: SOLR_SERVICE_HOST - value: &solr_host_port "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local:8983" - - name: SOLR_SERVICE_PORT - value: "8983" - - name: SOLR_LOCATION - value: *solr_host_port - - name: DATAVERSE_SOLR_HOST - value: "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local" - - name: DATAVERSE_SOLR_PORT - value: "8983" - - name: DATAVERSE_SOLR_CORE - value: dataverse + # Uppercase SOLR_* / DATAVERSE_SOLR_* are set by the chart when internalSolr.enabled (see templates/deployment.yaml). + # Duplicate keys here caused kubectl apply warnings ("hides previous definition"). Override only for external Solr. - name: dataverse_solr_host value: "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local" - name: dataverse_solr_port From 60c6f6ec7f2309c74f595a5941e29192896d312d Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 16:33:39 -0700 Subject: [PATCH 14/31] Automate the solr conf since we dont build images yet --- .github/workflows/deploy.yaml | 19 +++++ docs/HELM.md | 40 ++++++++++- ops/demo-deploy.tmpl.yaml | 38 +++++++++- scripts/k8s/ensure-solr-conf-configmap.sh | 74 ++++++++++++++++++++ scripts/solr-init-k8s.sh | 84 +++++++++++++++++++++++ 5 files changed, 250 insertions(+), 5 deletions(-) create mode 100755 scripts/k8s/ensure-solr-conf-configmap.sh create mode 100755 scripts/solr-init-k8s.sh diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 40e464a..f9976fd 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -104,6 +104,15 @@ jobs: ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT" + - name: Solr conf ConfigMap (pre-Helm) + env: + DV_REF: ${{ vars.DV_REF || 'v6.10.1' }} + SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }} + run: | + set -e + chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh + SOLR_RESTART_DEPLOYMENTS=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME" + - name: Deploy with Helm run: | set -e @@ -116,6 +125,16 @@ jobs: -l "app.kubernetes.io/instance=${HELM_RELEASE_NAME},app.kubernetes.io/name=${HELM_APP_NAME}" \ --timeout="${DEPLOY_ROLLOUT_TIMEOUT}" + - name: Solr workloads pick up ConfigMap updates + if: ${{ vars.SOLR_POST_HELM_ROLLOUT == 'true' }} + env: + DV_REF: ${{ vars.DV_REF || 'v6.10.1' }} + SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }} + run: | + set -e + chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh + SOLR_APPLY_CM=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME" + - name: One-shot bootstrap Job (non-hook) if: github.event_name == 'workflow_dispatch' && inputs.run_bootstrap_job run: | diff --git a/docs/HELM.md b/docs/HELM.md index 60c333b..d571f9f 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -125,8 +125,44 @@ The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo ro ## S3 file storage -1. Create a Secret in the release namespace with keys matching `awsS3.secretKeys` (default: `credentials`, `config`) — same shape as AWS CLI config files. -2. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `endpointUrl`, `region`, `profile`. +1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `endpointUrl`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. + +2. Create a **generic** Secret in the **same namespace** as the Helm release, **before** pods that mount it start. Key names must match `awsS3.secretKeys` (defaults below): the values are the **raw file contents** of `~/.aws/credentials` and `~/.aws/config`. + + - `credentials` — ini format; the profile block header (e.g. `[default]` or `[my-profile]`) must match **`awsS3.profile`**. + - `config` — ini format; for `profile: default` use `[default]` with `region = ...`. For a named profile use `[profile my-profile]` and the same region as `awsS3.region` unless you know you need otherwise. + +3. **Examples** (replace `NAMESPACE`, keys, region, and secret name if you changed `existingSecret`): + + ```sh + NS=NAMESPACE + kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - + + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-file=credentials="$HOME/.aws/credentials" \ + --from-file=config="$HOME/.aws/config" \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + Inline `[default]` user (no local files): + + ```sh + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-literal=credentials="[default] + aws_access_key_id = AKIA... + aws_secret_access_key = ... + " \ + --from-literal=config="[default] + region = us-west-2 + " \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + If you use **temporary credentials** (assumed role / STS), add a line to the credentials profile: `aws_session_token = ...`. Rotate before expiry or automate renewal. + +4. After creating or updating the Secret, **restart** the Dataverse Deployment (or delete its pods) so the volume is remounted. The chart sets `AWS_SHARED_CREDENTIALS_FILE` and `AWS_CONFIG_FILE` to the mounted paths. + +**Note:** The Java AWS SDK inside the app may not perform the same **assume-role chaining** as the AWS CLI from a complex `config` file. Prefer putting **direct** user keys or **already-assumed** temporary keys in the Secret for the app, or use EKS **IRSA** (service account + role) instead of long-lived keys if your platform supports it. ## Upgrades diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 1e6c51d..5abbb80 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -12,10 +12,42 @@ # # Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), # same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/HELM.md**). +# +# With **awsS3.enabled**, create a **generic** Secret in the release namespace **before** Helm installs pods that +# mount it. The chart mounts it as AWS CLI files under `/secrets/aws-cli/.aws/`. +# +# • Secret **metadata.name** must match **awsS3.existingSecret** below (default: aws-s3-credentials). +# • Secret **data keys** must match **awsS3.secretKeys** (default: `credentials` and `config`) — file bodies in +# the same format as `~/.aws/credentials` and `~/.aws/config`. +# • **awsS3.profile** must match a profile stanza in those files: `[default]` in credentials, and either +# `[default]` or `[profile yourname]` in config (see AWS CLI docs for named profiles). +# +# From your laptop (kubeconfig aimed at the cluster), examples — set NS to your namespace (e.g. demo-dataverseup): +# +# NS=demo-dataverseup +# kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +# +# # A) Reuse local CLI files (simplest if profiles already match awsS3.profile): +# kubectl -n "$NS" create secret generic aws-s3-credentials \ +# --from-file=credentials="$HOME/.aws/credentials" \ +# --from-file=config="$HOME/.aws/config" \ +# --dry-run=client -o yaml | kubectl apply -f - +# +# # B) Two small files (avoids multiline --from-literal quoting); region must match awsS3.region: +# printf '%s\n' '[default]' 'aws_access_key_id = AKIA...' 'aws_secret_access_key = ...' > /tmp/aws-credentials +# printf '%s\n' '[default]' 'region = us-west-2' > /tmp/aws-config +# kubectl -n "$NS" create secret generic aws-s3-credentials \ +# --from-file=credentials=/tmp/aws-credentials --from-file=config=/tmp/aws-config \ +# --dry-run=client -o yaml | kubectl apply -f && rm -f /tmp/aws-credentials /tmp/aws-config +# +# # C) STS / assumed-role: add `aws_session_token = ...` under the same profile in the credentials file body. +# +# After creating or rotating the Secret, restart the Dataverse Deployment so the volume remounts. +# Full detail: **docs/HELM.md** (S3 file storage). awsS3: enabled: true - # Name of a Secret you create out-of-band (keys = secretKeys below). + # Kubernetes Secret you create out-of-band (see comments above). existingSecret: "aws-s3-credentials" bucketName: "demo-dataverseup" endpointUrl: "https://s3.us-west-2.amazonaws.com" @@ -139,7 +171,7 @@ mail: # see ops/kubernetes-deploy-checklist.md — DB name here must exist on the server (e.g. demo-dataverseup). extraEnvVars: - name: DATAVERSE_DB_HOST - value: acid-postgres-cluster-delta.postgres.svc.cluster.local + value: postgresql.postgres.svc.cluster.local - name: DATAVERSE_DB_USER value: "demo-dataverseup" - name: DATAVERSE_DB_PASSWORD @@ -147,7 +179,7 @@ extraEnvVars: - name: DATAVERSE_DB_NAME value: "demo-dataverseup" - name: POSTGRES_SERVER - value: acid-postgres-cluster-delta.postgres.svc.cluster.local + value: postgresql.postgres.svc.cluster.local - name: POSTGRES_PORT value: "5432" - name: POSTGRES_DATABASE diff --git a/scripts/k8s/ensure-solr-conf-configmap.sh b/scripts/k8s/ensure-solr-conf-configmap.sh new file mode 100755 index 0000000..aaaf8ad --- /dev/null +++ b/scripts/k8s/ensure-solr-conf-configmap.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Build IQSS Dataverse Solr config (same layout as test-k3d-deploy/k3d/install.sh) and apply ConfigMap +# with key solr-conf.tgz — required by the chart's internal Solr init container. +# +# Uses curl + tar only (no Docker). Run after kubectl context points at the target cluster. +# +# Usage: ./scripts/k8s/ensure-solr-conf-configmap.sh NAMESPACE +# Env: +# DV_REF IQSS Dataverse tag (default: v6.10.1) +# SOLR_DIST_VERSION Apache Solr release for _default configset extras (default: 9.10.1) +# CONFIGMAP_NAME (default: dataverse-solr-conf) +set -eu + +NAMESPACE="${1:-}" +if [ -z "$NAMESPACE" ]; then + echo "usage: $0 NAMESPACE" >&2 + exit 1 +fi + +DV_REF="${DV_REF:-v6.10.1}" +SOLR_DIST_VERSION="${SOLR_DIST_VERSION:-9.10.1}" +CONFIGMAP_NAME="${CONFIGMAP_NAME:-dataverse-solr-conf}" + +for cmd in kubectl curl tar; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "missing required command: $cmd" >&2 + exit 1 + fi +done + +TMP="$(mktemp -d)" +cleanup() { + rm -rf "$TMP" +} +trap cleanup EXIT + +echo "Fetching IQSS Dataverse ${DV_REF} (conf/solr) ..." +curl -fsSL "https://codeload.github.com/IQSS/dataverse/tar.gz/${DV_REF}" -o "$TMP/dv.tgz" +tar -xzf "$TMP/dv.tgz" -C "$TMP" +TOP="$(find "$TMP" -maxdepth 1 -type d -name 'dataverse-*' | head -1)" +if [ -z "$TOP" ] || [ ! -d "$TOP/conf/solr" ]; then + echo "Could not find dataverse-*/conf/solr in archive (try DV_REF=develop or a valid tag)." >&2 + exit 1 +fi + +STAGE="${TMP}/solr-stage" +mkdir -p "${STAGE}" +cp -a "${TOP}/conf/solr/." "${STAGE}/" + +SPREFIX="solr-${SOLR_DIST_VERSION}/server/solr/configsets/_default/conf" +SOLR_URL="https://archive.apache.org/dist/solr/solr/${SOLR_DIST_VERSION}/solr-${SOLR_DIST_VERSION}.tgz" +echo "Fetching Solr ${SOLR_DIST_VERSION} _default conf companions from ${SOLR_URL} ..." +curl -fsSL "${SOLR_URL}" -o "$TMP/apache-solr.tgz" +tar -xzf "$TMP/apache-solr.tgz" -C "$TMP" \ + "${SPREFIX}/lang" \ + "${SPREFIX}/protwords.txt" \ + "${SPREFIX}/stopwords.txt" \ + "${SPREFIX}/synonyms.txt" + +CONFROOT="${TMP}/${SPREFIX}" +cp -a "${CONFROOT}/lang" "${STAGE}/" +for f in protwords.txt stopwords.txt synonyms.txt; do + cp -a "${CONFROOT}/${f}" "${STAGE}/${f}" +done + +SOLR_TGZ="${TMP}/solr-conf.tgz" +tar -czf "${SOLR_TGZ}" -C "${STAGE}" . +echo "Packaged Solr conf ($(du -h "${SOLR_TGZ}" | cut -f1)) → ConfigMap ${CONFIGMAP_NAME} key solr-conf.tgz" + +kubectl -n "${NAMESPACE}" create configmap "${CONFIGMAP_NAME}" \ + --from-file=solr-conf.tgz="${SOLR_TGZ}" \ + --dry-run=client -o yaml | kubectl apply -f - + +echo "Applied ConfigMap ${CONFIGMAP_NAME} in namespace ${NAMESPACE}" diff --git a/scripts/solr-init-k8s.sh b/scripts/solr-init-k8s.sh new file mode 100755 index 0000000..b84ddbf --- /dev/null +++ b/scripts/solr-init-k8s.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# Apply Dataverse Solr conf ConfigMap to a namespace and optionally restart in-chart workloads so +# init containers / Solr pick up new conf (e.g. after IQSS schema changes or first-time bootstrap). +# +# Requires: kubectl, curl, tar (same as scripts/k8s/ensure-solr-conf-configmap.sh). +# +# Usage (from repo root): +# ./scripts/solr-init-k8s.sh NAMESPACE HELM_RELEASE_NAME +# +# Env: +# SOLR_APPLY_CM=true|false Apply ConfigMap (default: true) +# SOLR_RESTART_DEPLOYMENTS=true|false Rollout restart matching Deployments (default: true) +# CHART_APP_NAME app.kubernetes.io/name base label (default: dataverseup) +# DV_REF, SOLR_DIST_VERSION, CONFIGMAP_NAME — forwarded to ensure-solr-conf-configmap.sh +# SOLR_ROLLOUT_TIMEOUT e.g. 5m (default: 8m) +set -eu + +NS="${1:-}" +REL="${2:-}" +if [ -z "$NS" ] || [ -z "$REL" ]; then + echo "usage: $0 NAMESPACE HELM_RELEASE_NAME" >&2 + echo " example: $0 demo-dataverseup demo-dataverseup" >&2 + exit 1 +fi + +SOLR_APPLY_CM="${SOLR_APPLY_CM:-true}" +SOLR_RESTART_DEPLOYMENTS="${SOLR_RESTART_DEPLOYMENTS:-true}" +CHART_APP_NAME="${CHART_APP_NAME:-${HELM_APP_NAME:-dataverseup}}" +SOLR_ROLLOUT_TIMEOUT="${SOLR_ROLLOUT_TIMEOUT:-8m}" + +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +ENSURE="${ROOT}/scripts/k8s/ensure-solr-conf-configmap.sh" + +if [ -f "$ENSURE" ] && [ ! -x "$ENSURE" ]; then + chmod +x "$ENSURE" || true +fi +if [ ! -f "$ENSURE" ]; then + echo "missing ${ENSURE}" >&2 + exit 1 +fi + +command -v kubectl >/dev/null 2>&1 || { + echo "kubectl is required" >&2 + exit 1 +} + +_truthy() { + case "$1" in + 1 | true | TRUE | yes | YES) return 0 ;; + *) return 1 ;; + esac +} + +if _truthy "$SOLR_APPLY_CM"; then + echo "=== Solr: applying ConfigMap (${CONFIGMAP_NAME:-dataverse-solr-conf}) ===" + "$ENSURE" "$NS" +else + echo "=== Solr: skipping ConfigMap apply (SOLR_APPLY_CM=false) ===" +fi + +if ! _truthy "$SOLR_RESTART_DEPLOYMENTS"; then + echo "=== Solr: skipping rollout restarts (SOLR_RESTART_DEPLOYMENTS=false) ===" + exit 0 +fi + +SOLR_LABEL="app.kubernetes.io/instance=${REL},app.kubernetes.io/name=${CHART_APP_NAME}-solr" +APP_LABEL="app.kubernetes.io/instance=${REL},app.kubernetes.io/name=${CHART_APP_NAME}" + +restart_labeled() { + _label="$1" + _kind="$2" + kubectl -n "$NS" get deploy -l "$_label" -o name 2>/dev/null | while read -r res; do + [ -z "$res" ] && continue + echo "=== Solr: rollout restart $_kind ($res) ===" + kubectl -n "$NS" rollout restart "$res" + kubectl -n "$NS" rollout status "$res" --timeout="$SOLR_ROLLOUT_TIMEOUT" + done +} + +# Solr first (Dataverse init container waits for core ping). +restart_labeled "$SOLR_LABEL" "internal Solr" +restart_labeled "$APP_LABEL" "Dataverse" + +echo "=== Solr: done ===" From 07aaf4374e8eac4c481231834afa471bb478d1dc Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 16:39:34 -0700 Subject: [PATCH 15/31] Update the postgers host name correctly --- ops/demo-deploy.tmpl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 5abbb80..6f599cc 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -171,7 +171,7 @@ mail: # see ops/kubernetes-deploy-checklist.md — DB name here must exist on the server (e.g. demo-dataverseup). extraEnvVars: - name: DATAVERSE_DB_HOST - value: postgresql.postgres.svc.cluster.local + value: postgres-postgresql.postgres.svc.cluster.local - name: DATAVERSE_DB_USER value: "demo-dataverseup" - name: DATAVERSE_DB_PASSWORD @@ -179,7 +179,7 @@ extraEnvVars: - name: DATAVERSE_DB_NAME value: "demo-dataverseup" - name: POSTGRES_SERVER - value: postgresql.postgres.svc.cluster.local + value: postgres-postgresql.postgres.svc.cluster.local - name: POSTGRES_PORT value: "5432" - name: POSTGRES_DATABASE From 86f074b2d1703d2b40f417624feebcb313928e13 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 16:45:42 -0700 Subject: [PATCH 16/31] add init setup to charts for automation --- .../dataverseup/files/init.d/006-s3-aws-storage.sh | 1 + charts/dataverseup/files/init.d/01-persistent-id.sh | 1 + charts/dataverseup/files/init.d/010-languages.sh | 1 + charts/dataverseup/files/init.d/010-mailrelay-set.sh | 1 + charts/dataverseup/files/init.d/011-local-storage.sh | 1 + charts/dataverseup/files/init.d/012-minio-bucket1.sh | 1 + charts/dataverseup/files/init.d/013-minio-bucket2.sh | 1 + charts/dataverseup/files/init.d/02-controlled-voc.sh | 1 + charts/dataverseup/files/init.d/03-doi-set.sh | 1 + charts/dataverseup/files/init.d/04-setdomain.sh | 1 + charts/dataverseup/files/init.d/05-reindex.sh | 1 + charts/dataverseup/files/init.d/07-previewers.sh | 1 + .../dataverseup/files/init.d/08-federated-login.sh | 1 + charts/dataverseup/files/init.d/1001-webhooks.sh | 1 + .../dataverseup/files/init.d/1002-custom-metadata.sh | 1 + charts/dataverseup/templates/configmap.yaml | 12 +++++++++++- charts/dataverseup/templates/deployment.yaml | 8 ++++---- charts/dataverseup/values.yaml | 6 ++++++ docs/HELM.md | 2 ++ ops/demo-deploy.tmpl.yaml | 7 +++++++ 20 files changed, 45 insertions(+), 5 deletions(-) create mode 120000 charts/dataverseup/files/init.d/006-s3-aws-storage.sh create mode 120000 charts/dataverseup/files/init.d/01-persistent-id.sh create mode 120000 charts/dataverseup/files/init.d/010-languages.sh create mode 120000 charts/dataverseup/files/init.d/010-mailrelay-set.sh create mode 120000 charts/dataverseup/files/init.d/011-local-storage.sh create mode 120000 charts/dataverseup/files/init.d/012-minio-bucket1.sh create mode 120000 charts/dataverseup/files/init.d/013-minio-bucket2.sh create mode 120000 charts/dataverseup/files/init.d/02-controlled-voc.sh create mode 120000 charts/dataverseup/files/init.d/03-doi-set.sh create mode 120000 charts/dataverseup/files/init.d/04-setdomain.sh create mode 120000 charts/dataverseup/files/init.d/05-reindex.sh create mode 120000 charts/dataverseup/files/init.d/07-previewers.sh create mode 120000 charts/dataverseup/files/init.d/08-federated-login.sh create mode 120000 charts/dataverseup/files/init.d/1001-webhooks.sh create mode 120000 charts/dataverseup/files/init.d/1002-custom-metadata.sh diff --git a/charts/dataverseup/files/init.d/006-s3-aws-storage.sh b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh new file mode 120000 index 0000000..9c9b8ab --- /dev/null +++ b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh @@ -0,0 +1 @@ +../../../../init.d/006-s3-aws-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/01-persistent-id.sh b/charts/dataverseup/files/init.d/01-persistent-id.sh new file mode 120000 index 0000000..0d76aa3 --- /dev/null +++ b/charts/dataverseup/files/init.d/01-persistent-id.sh @@ -0,0 +1 @@ +../../../../init.d/01-persistent-id.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-languages.sh b/charts/dataverseup/files/init.d/010-languages.sh new file mode 120000 index 0000000..a71e6e7 --- /dev/null +++ b/charts/dataverseup/files/init.d/010-languages.sh @@ -0,0 +1 @@ +../../../../init.d/010-languages.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-mailrelay-set.sh b/charts/dataverseup/files/init.d/010-mailrelay-set.sh new file mode 120000 index 0000000..da144d5 --- /dev/null +++ b/charts/dataverseup/files/init.d/010-mailrelay-set.sh @@ -0,0 +1 @@ +../../../../init.d/010-mailrelay-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/011-local-storage.sh b/charts/dataverseup/files/init.d/011-local-storage.sh new file mode 120000 index 0000000..e16e2d0 --- /dev/null +++ b/charts/dataverseup/files/init.d/011-local-storage.sh @@ -0,0 +1 @@ +../../../../init.d/011-local-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/012-minio-bucket1.sh b/charts/dataverseup/files/init.d/012-minio-bucket1.sh new file mode 120000 index 0000000..9c70fdf --- /dev/null +++ b/charts/dataverseup/files/init.d/012-minio-bucket1.sh @@ -0,0 +1 @@ +../../../../init.d/012-minio-bucket1.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/013-minio-bucket2.sh b/charts/dataverseup/files/init.d/013-minio-bucket2.sh new file mode 120000 index 0000000..775442a --- /dev/null +++ b/charts/dataverseup/files/init.d/013-minio-bucket2.sh @@ -0,0 +1 @@ +../../../../init.d/013-minio-bucket2.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/02-controlled-voc.sh b/charts/dataverseup/files/init.d/02-controlled-voc.sh new file mode 120000 index 0000000..816a86c --- /dev/null +++ b/charts/dataverseup/files/init.d/02-controlled-voc.sh @@ -0,0 +1 @@ +../../../../init.d/02-controlled-voc.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/03-doi-set.sh b/charts/dataverseup/files/init.d/03-doi-set.sh new file mode 120000 index 0000000..0d02c60 --- /dev/null +++ b/charts/dataverseup/files/init.d/03-doi-set.sh @@ -0,0 +1 @@ +../../../../init.d/03-doi-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/04-setdomain.sh b/charts/dataverseup/files/init.d/04-setdomain.sh new file mode 120000 index 0000000..791a335 --- /dev/null +++ b/charts/dataverseup/files/init.d/04-setdomain.sh @@ -0,0 +1 @@ +../../../../init.d/04-setdomain.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/05-reindex.sh b/charts/dataverseup/files/init.d/05-reindex.sh new file mode 120000 index 0000000..c256126 --- /dev/null +++ b/charts/dataverseup/files/init.d/05-reindex.sh @@ -0,0 +1 @@ +../../../../init.d/05-reindex.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/07-previewers.sh b/charts/dataverseup/files/init.d/07-previewers.sh new file mode 120000 index 0000000..804ac47 --- /dev/null +++ b/charts/dataverseup/files/init.d/07-previewers.sh @@ -0,0 +1 @@ +../../../../init.d/07-previewers.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/08-federated-login.sh b/charts/dataverseup/files/init.d/08-federated-login.sh new file mode 120000 index 0000000..43463b7 --- /dev/null +++ b/charts/dataverseup/files/init.d/08-federated-login.sh @@ -0,0 +1 @@ +../../../../init.d/08-federated-login.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1001-webhooks.sh b/charts/dataverseup/files/init.d/1001-webhooks.sh new file mode 120000 index 0000000..66eaefe --- /dev/null +++ b/charts/dataverseup/files/init.d/1001-webhooks.sh @@ -0,0 +1 @@ +../../../../init.d/1001-webhooks.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1002-custom-metadata.sh b/charts/dataverseup/files/init.d/1002-custom-metadata.sh new file mode 120000 index 0000000..05b8703 --- /dev/null +++ b/charts/dataverseup/files/init.d/1002-custom-metadata.sh @@ -0,0 +1 @@ +../../../../init.d/1002-custom-metadata.sh \ No newline at end of file diff --git a/charts/dataverseup/templates/configmap.yaml b/charts/dataverseup/templates/configmap.yaml index 41bc6a8..51a6d11 100644 --- a/charts/dataverseup/templates/configmap.yaml +++ b/charts/dataverseup/templates/configmap.yaml @@ -1,4 +1,4 @@ -{{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled }} +{{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -6,6 +6,15 @@ metadata: labels: {{- include "dataverseup.labels" . | nindent 4 }} data: + {{- if .Values.initdFromChart.enabled }} + {{- range $path, $bytes := .Files.Glob "files/init.d/*.sh" }} + {{ base $path }}: | +{{- $bytes | toString | trim | nindent 4 }} + {{- end }} + {{- if .Values.configMap.enabled }} + {{- toYaml .Values.configMap.data | nindent 2 }} + {{- end }} + {{- else }} {{- if .Values.configMap.enabled }} {{- toYaml .Values.configMap.data | nindent 2 }} {{- end }} @@ -19,4 +28,5 @@ data: 006-s3-aws-storage.sh: | {{- $s3Script | nindent 4 }} {{- end }} + {{- end }} {{- end }} diff --git a/charts/dataverseup/templates/deployment.yaml b/charts/dataverseup/templates/deployment.yaml index 700584a..909b996 100644 --- a/charts/dataverseup/templates/deployment.yaml +++ b/charts/dataverseup/templates/deployment.yaml @@ -182,7 +182,7 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumeMounts }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled .Values.volumeMounts }} volumeMounts: {{- if .Values.persistence.enabled }} - name: data @@ -195,7 +195,7 @@ spec: - name: branding-writable-docroot mountPath: /dv/docroot {{- end }} - {{- if or .Values.configMap.enabled .Values.mail.enabled }} + {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.initdFromChart.enabled }} - name: init-d mountPath: {{ .Values.configMap.mountPath }} readOnly: true @@ -213,7 +213,7 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} - {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.volumes .Values.solrInit.enabled }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled .Values.volumes .Values.solrInit.enabled }} volumes: {{- if .Values.persistence.enabled }} - name: data @@ -238,7 +238,7 @@ spec: emptyDir: {} {{- end }} {{- end }} - {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled }} + {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled }} - name: init-d configMap: name: {{ include "dataverseup.fullname" . }}-config diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index 9e333a3..0afa5aa 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -153,6 +153,12 @@ brandingNavbarLogos: docrootUid: 1000 docrootGid: 1000 +# Ship the repo **init.d/*.sh** bundle into the Payara ConfigMap (compose parity: ./init.d → /opt/payara/init.d). +# When true, all scripts under charts/dataverseup/files/init.d/*.sh are included; optional configMap.data overlays. +# Some scripts assume compose paths (MinIO, /opt/payara/triggers, etc.) — review before enabling in production. +initdFromChart: + enabled: false + # Optional ConfigMap created by this chart; keys become filenames under mountPath (e.g. init scripts). configMap: enabled: false diff --git a/docs/HELM.md b/docs/HELM.md index d571f9f..af7e903 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -123,6 +123,8 @@ If you terminate TLS or expose the app on a **non-default host port**, keep **`D The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. +Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting `./init.d`). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and triggers-specific scripts before enabling in a cluster that does not mount those paths. + ## S3 file storage 1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `endpointUrl`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 6f599cc..1135100 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -163,6 +163,13 @@ solrInit: adminPassword: "" resources: {} +# Compose mounts the full repo **init.d/** at `/opt/payara/init.d`. Enable this to ship the same `*.sh` bundle via +# the chart ConfigMap (see **initdFromChart** in **charts/dataverseup/values.yaml**). Scripts run in lexical order; +# **012/013-minio** expect compose MinIO (no-op or skip if you use S3 only). **1001-webhooks** needs **triggers/** +# on disk unless you mount it separately. +initdFromChart: + enabled: true + # Mount init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. mail: enabled: true From 21a792727ef7b5472b7c518427d415e07f0a6e31 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 16:56:10 -0700 Subject: [PATCH 17/31] automate the application of the branded elements --- docs/HELM.md | 10 ++++++++++ ops/demo-deploy.tmpl.yaml | 21 ++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/HELM.md b/docs/HELM.md index af7e903..e8c79e9 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -28,6 +28,16 @@ HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_ - **Optional dedicated Solr** (`internalSolr`) — a **new** Solr Deployment/Service in the **same release and namespace** as Dataverse (not wiring into someone else’s shared “cluster Solr”). Default **`solrInit.mode`** is **`standalone`**: the Dataverse pod waits for that Solr core before starting. Use **`solrInit.mode: cloud`** only when Dataverse talks to **SolrCloud + ZooKeeper** you operate separately. - **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. +### Branding (navbar logo + Admin API settings) + +1. **Navbar SVG** — Enable **`brandingNavbarLogos.enabled`** so an init container copies **`branding/docroot/logos/navbar/logo.svg`** from the chart onto **`/dv/docroot/logos/navbar/logo.svg`** (needs **`docrootPersistence`** or the chart’s emptyDir docroot fallback). Match **`LOGO_CUSTOMIZATION_FILE`** in **`branding/branding.env`** to the web path (e.g. `/logos/navbar/logo.svg`). + +2. **Admin settings** (installation name, footer, optional custom header/footer CSS paths) — Edit **`branding/branding.env`** in the repo. The chart embeds it in the **`…-bootstrap-chain`** ConfigMap when **`bootstrapJob.mode: compose`**. The post-install Job runs **`apply-branding.sh`**, which PUTs those settings via the Dataverse Admin API using the admin token from configbaker. + +3. **Custom HTML/CSS files** — Add them under **`branding/docroot/branding/`** in the repo, set **`HEADER_CUSTOMIZATION_FILE`**, etc. in **`branding.env`** to **`/dv/docroot/branding/...`**, and ship those files into the pod (extra **`volumeMounts`** / **`configMap`** or bake into an image). The stock chart does not mount the whole **`branding/docroot/branding/`** tree on the main Deployment; compose only ships **`branding.env`** and the logo via **`brandingNavbarLogos`**. + +4. **After `helm upgrade`** — The post-install hook does **not** re-run. To re-apply branding, use **`bootstrapJob.compose.postUpgradeBrandingSeedJob`** with a Secret holding **`DATAVERSE_API_TOKEN`**, or run **`scripts/apply-branding.sh`** locally/cron with **`DATAVERSE_INTERNAL_URL`** and a token. + The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). ### Recommended Solr layout: new instance with this deploy diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 1135100..62ecef3 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -102,6 +102,14 @@ docrootPersistence: storageClassName: "" existingClaim: "" +# Navbar logo file on disk (chart embeds **branding/docroot/logos/navbar/logo.svg**). Init copies it to +# `/dv/docroot/logos/navbar/logo.svg` so it matches **LOGO_CUSTOMIZATION_FILE** in **branding/branding.env** +# (API path `/logos/navbar/logo.svg`). For custom header/footer/CSS, add files under **branding/docroot/branding/** +# in the repo, extend **branding.env** with `/dv/docroot/branding/...` paths, then rely on **bootstrapJob.mode: compose** +# (below) to PUT those settings after configbaker — or run **scripts/apply-branding.sh** manually with an admin token. +brandingNavbarLogos: + enabled: true + ingress: enabled: true className: nginx-ingress @@ -278,10 +286,17 @@ extraEnvVars: value: "true" # gdcc/configbaker after install (post-install hook). Same tag as image.tag above. -# Does not run on `helm upgrade`. After the DB is bootstrapped once, you may set enabled: false to avoid -# the hook re-running on a future `helm uninstall` + `helm install` against the same database. -# If you wiped Postgres later, use GitHub Deploy → "Run configbaker bootstrap job" or apply a Job by hand. +# **mode: compose** — same order as compose: configbaker → **apply-branding.sh** (reads **branding/branding.env** +# baked into chart) → optional **seed-content** (fixtures). Does not run on plain `helm upgrade` when helmHook=true. +# Re-apply branding after changing **branding/**: use workflow "Run bootstrap job" with hook disabled, or +# **postUpgradeBrandingSeedJob** + admin API token Secret (see **charts/dataverseup/values.yaml**). +# If you wiped Postgres later, use GitHub Deploy → "Run bootstrap job" or apply a Job by hand. bootstrapJob: enabled: true + mode: compose helmHook: true timeout: 20m + compose: + waitMaxSeconds: 900 + waitSleepSeconds: 5 + seed: true From 2403a5f22356dd7f96cd8fe656e1c3b805007111 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 17:37:45 -0700 Subject: [PATCH 18/31] Update script for applying branding to correct location on k8 --- scripts/apply-branding.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/apply-branding.sh b/scripts/apply-branding.sh index 545e416..669a37a 100755 --- a/scripts/apply-branding.sh +++ b/scripts/apply-branding.sh @@ -42,17 +42,25 @@ if [ -n "${FOOTER_COPYRIGHT:-}" ]; then esac fi +# Path segment must be URL-encoded (at least ':') so proxies like nginx do not mishandle +# /api/admin/settings/:LogoCustomizationFile as a bogus port or split path. +admin_setting_path() { + printf '%s' "$1" | sed -e 's/%/%25/g' -e 's/:/%3A/g' -e 's/ /%20/g' +} + curl_put_setting() { _name="$1" _val="$2" if [ -z "$_val" ]; then return 0 fi + _path_seg=$(admin_setting_path "$_name") printf '%s' "apply-branding: PUT ${_name}\n" >&2 - _code=$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ + _code=$(curl -sS -g -o /dev/null -w "%{http_code}" -X PUT \ -H "X-Dataverse-key: ${TOKEN}" \ + -H "Content-Type: text/plain; charset=UTF-8" \ --data-binary "$_val" \ - "${API}/admin/settings/${_name}") + "${API}/admin/settings/${_path_seg}") if [ "$_code" != "200" ] && [ "$_code" != "204" ]; then echo "apply-branding: WARNING ${_name} -> HTTP ${_code} (check admin user API token in secrets/api/key)" >&2 fi From e0646505edc44268699fe46b3f625dd06f76f1bb Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 17:55:59 -0700 Subject: [PATCH 19/31] update the brandin.env to have correct info --- branding/branding.env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/branding/branding.env b/branding/branding.env index 47f207c..3c576b8 100644 --- a/branding/branding.env +++ b/branding/branding.env @@ -3,7 +3,7 @@ # For :HeaderCustomizationFile / :FooterCustomizationFile / :StyleCustomizationFile use *filesystem* paths in the # API, e.g. /dv/docroot/branding/custom-header.html (see branding.env examples below). -# INSTALLATION_NAME='My Repository' +INSTALLATION_NAME='DataverseUp' # Navbar logo file on disk: branding/docroot/logos/navbar/logo.svg → URL below (must match filename). LOGO_CUSTOMIZATION_FILE=/logos/navbar/logo.svg @@ -21,8 +21,8 @@ DISABLE_ROOT_DATAVERSE_THEME=true # Multi-word values must be quoted. FOOTER_COPYRIGHT is appended to the built-in "Copyright © YEAR" with no # separator in the UI; apply-branding.sh prepends a space unless you start with space/tab, —, -, |, or (. -FOOTER_COPYRIGHT='Notch8' -# NAVBAR_ABOUT_URL=https://example.edu/about -# NAVBAR_SUPPORT_URL=mailto:support@example.edu -# NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/ +FOOTER_COPYRIGHT=' DataverseUp | Powered by Notch8, your partner in digital preservation.' +NAVBAR_ABOUT_URL=https://notch8.com/about +NAVBAR_SUPPORT_URL=mailto:support@notch8.com +NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/ From c7f6d991a05dd24ab5704c403e1e1fd4eeb28bd3 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 18:00:27 -0700 Subject: [PATCH 20/31] Update to compose to apply branding step each time --- .github/workflows/deploy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f9976fd..74eba3f 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -147,6 +147,7 @@ jobs: --show-only templates/bootstrap-job.yaml \ --set bootstrapJob.enabled=true \ --set bootstrapJob.helmHook=false \ + --set bootstrapJob.mode=compose \ >"$OUT" JOB="$(awk '/^kind: Job$/{j=1} j && /^ name: /{print $2; exit}' "$OUT")" if [ -z "$JOB" ]; then From cd997cb494309ed6d7f8a9a74cf8e13c6ddee537 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 18:30:45 -0700 Subject: [PATCH 21/31] Update the job to have long lived token through secrets --- .github/workflows/deploy.yaml | 3 ++- .../dataverseup/templates/bootstrap-job.yaml | 7 ++++++ charts/dataverseup/values.yaml | 5 ++++ ops/demo-deploy.tmpl.yaml | 6 +++++ scripts/k8s-bootstrap-chain.sh | 23 +++++++++++++++---- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 74eba3f..e9aea9d 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -27,7 +27,7 @@ on: required: false type: string run_bootstrap_job: - description: Re-run the bootstrap Job (helmHook=false) — same manifest as chart templates/bootstrap-job.yaml; use if the post-install hook was skipped or you reset Postgres. Chart default mode is oneShot (configbaker bootstrap.sh dev). Deletes the prior Job by name first. + description: Re-run the bootstrap Job (helmHook=false, compose). Requires Secret dataverse-admin-api-token (key token) in the namespace when the instance is already bootstrapped — see ops/*-deploy.tmpl.yaml bootstrapJob.compose.existingAdminApiTokenSecret. Deletes the prior Job by name first. required: false type: boolean default: false @@ -144,6 +144,7 @@ jobs: helm template "$HELM_RELEASE_NAME" "$HELM_CHART_PATH" \ --namespace "$HELM_NAMESPACE" \ $HELM_EXTRA_ARGS \ + --show-only templates/bootstrap-chain-configmap.yaml \ --show-only templates/bootstrap-job.yaml \ --set bootstrapJob.enabled=true \ --set bootstrapJob.helmHook=false \ diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml index c4eb117..ee8af8f 100644 --- a/charts/dataverseup/templates/bootstrap-job.yaml +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -53,6 +53,13 @@ spec: value: "/config/branding.env" - name: TIMEOUT value: {{ .Values.bootstrapJob.timeout | quote }} + {{- if .Values.bootstrapJob.compose.existingAdminApiTokenSecret }} + - name: DATAVERSE_API_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.bootstrapJob.compose.existingAdminApiTokenSecret | quote }} + key: {{ .Values.bootstrapJob.compose.adminApiTokenSecretKey | default "token" | quote }} + {{- end }} volumeMounts: - name: bootstrap-scripts mountPath: /bootstrap-chain diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index 0afa5aa..a782053 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -308,6 +308,11 @@ bootstrapJob: waitSleepSeconds: 5 # Run fixtures/seed after branding (set false for bootstrap+branding only). seed: true + # When the DB is already bootstrapped, configbaker prints "skipping" and may not write API_TOKEN to the file. + # For CI "re-run bootstrap job" / branding+seed, create a Secret with a superuser API token and set the name: + # kubectl create secret generic dataverse-admin-api-token -n YOUR_NS --from-literal=token=UUID + existingAdminApiTokenSecret: "" + adminApiTokenSecretKey: token # post-upgrade hook: branding + seed only (no configbaker). Use when the release was upgraded and the # post-install hook never ran compose, or you need to re-apply after changing branding/fixtures in the chart. # Create a Secret first, e.g. kubectl create secret generic dv-admin-api-token -n ns --from-literal=token=UUID diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 62ecef3..f515820 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -291,6 +291,10 @@ extraEnvVars: # Re-apply branding after changing **branding/**: use workflow "Run bootstrap job" with hook disabled, or # **postUpgradeBrandingSeedJob** + admin API token Secret (see **charts/dataverseup/values.yaml**). # If you wiped Postgres later, use GitHub Deploy → "Run bootstrap job" or apply a Job by hand. +# +# When Dataverse is **already bootstrapped**, configbaker skips and does not write **API_TOKEN** — the chain then +# needs a long-lived superuser API token from a Secret (same as post-upgrade branding job): +# kubectl -n demo-dataverseup create secret generic dataverse-admin-api-token --from-literal=token=YOUR_UUID bootstrapJob: enabled: true mode: compose @@ -300,3 +304,5 @@ bootstrapJob: waitMaxSeconds: 900 waitSleepSeconds: 5 seed: true + existingAdminApiTokenSecret: dataverse-admin-api-token + adminApiTokenSecretKey: token diff --git a/scripts/k8s-bootstrap-chain.sh b/scripts/k8s-bootstrap-chain.sh index 8742baa..bca1e4b 100755 --- a/scripts/k8s-bootstrap-chain.sh +++ b/scripts/k8s-bootstrap-chain.sh @@ -70,13 +70,28 @@ else fi "${CONFIGBAKER_BOOTSTRAP}" -e "${TOKEN_FILE}" dev + # Kubernetes may inject DATAVERSE_API_TOKEN via secretKeyRef; configbaker's bootstrap.env can still + # assign empty vars — sourcing must not wipe that token. + DATAVERSE_API_TOKEN_PRE_SOURCE="${DATAVERSE_API_TOKEN:-}" + # shellcheck disable=SC1090 - source "${TOKEN_FILE}" - if [[ -z "${API_TOKEN:-}" ]]; then - echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} after bootstrap" >&2 + set +e + [[ -f "${TOKEN_FILE}" ]] && source "${TOKEN_FILE}" + set -e + + if [[ -n "${API_TOKEN:-}" ]]; then + write_api_key_from_token "${API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} (configbaker often skips when already bootstrapped); using DATAVERSE_API_TOKEN from environment" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN_PRE_SOURCE:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE}; bootstrap.env cleared DATAVERSE_API_TOKEN — using token from environment (e.g. Secret) before source" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN_PRE_SOURCE}" + else + echo "k8s-bootstrap-chain: no API_TOKEN after bootstrap and no DATAVERSE_API_TOKEN in environment." >&2 + echo "k8s-bootstrap-chain: fix: create a Secret with a superuser API token and set bootstrapJob.compose.existingAdminApiTokenSecret in Helm values (see ops/demo-deploy.tmpl.yaml), or run with BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP=1 and DATAVERSE_API_TOKEN." >&2 exit 2 fi - write_api_key_from_token "${API_TOKEN}" fi export BRANDING_ENV_PATH="${BRANDING_ENV_PATH:-/config/branding.env}" From 7f9b4d20b7c371e9e05240bdc7e785b8def5f4a4 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 18:39:16 -0700 Subject: [PATCH 22/31] Allow optional --- charts/dataverseup/templates/bootstrap-job.yaml | 1 + charts/dataverseup/values.yaml | 4 +++- ops/demo-deploy.tmpl.yaml | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml index ee8af8f..e3f989a 100644 --- a/charts/dataverseup/templates/bootstrap-job.yaml +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -59,6 +59,7 @@ spec: secretKeyRef: name: {{ .Values.bootstrapJob.compose.existingAdminApiTokenSecret | quote }} key: {{ .Values.bootstrapJob.compose.adminApiTokenSecretKey | default "token" | quote }} + optional: true {{- end }} volumeMounts: - name: bootstrap-scripts diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index a782053..3104907 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -309,7 +309,9 @@ bootstrapJob: # Run fixtures/seed after branding (set false for bootstrap+branding only). seed: true # When the DB is already bootstrapped, configbaker prints "skipping" and may not write API_TOKEN to the file. - # For CI "re-run bootstrap job" / branding+seed, create a Secret with a superuser API token and set the name: + # Optional admin API token Secret (same name can stay in values for the whole lifecycle). The Job uses + # secretKeyRef optional:true so a missing Secret on first install does not block the pod — configbaker + # still writes API_TOKEN. After bootstrap, create the Secret for re-runs / already-bootstrapped DB: # kubectl create secret generic dataverse-admin-api-token -n YOUR_NS --from-literal=token=UUID existingAdminApiTokenSecret: "" adminApiTokenSecretKey: token diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index f515820..ab95935 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -292,8 +292,8 @@ extraEnvVars: # **postUpgradeBrandingSeedJob** + admin API token Secret (see **charts/dataverseup/values.yaml**). # If you wiped Postgres later, use GitHub Deploy → "Run bootstrap job" or apply a Job by hand. # -# When Dataverse is **already bootstrapped**, configbaker skips and does not write **API_TOKEN** — the chain then -# needs a long-lived superuser API token from a Secret (same as post-upgrade branding job): +# **existingAdminApiTokenSecret** is optional on first install (chart uses optional secretKeyRef). Create the +# Secret after you have a superuser API token so re-runs / already-bootstrapped DB work: # kubectl -n demo-dataverseup create secret generic dataverse-admin-api-token --from-literal=token=YOUR_UUID bootstrapJob: enabled: true From 2b74989c82a1979032041bb080aff733a4ecf522 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 18:58:42 -0700 Subject: [PATCH 23/31] Update the script to add the token to pass correctly and the CHAIN_SCRIPT_REVISION to see if it sets the correct one --- scripts/k8s-bootstrap-chain.sh | 85 ++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/scripts/k8s-bootstrap-chain.sh b/scripts/k8s-bootstrap-chain.sh index bca1e4b..d2ac020 100755 --- a/scripts/k8s-bootstrap-chain.sh +++ b/scripts/k8s-bootstrap-chain.sh @@ -3,6 +3,10 @@ # Runs inside gdcc/configbaker (bootstrap.sh) then applies branding + seed via curl scripts. set -euo pipefail +# Bump when token / configbaker behavior changes (grep logs for this to confirm the cluster mounted an updated ConfigMap). +CHAIN_SCRIPT_REVISION=4 +echo "k8s-bootstrap-chain: chain script revision=${CHAIN_SCRIPT_REVISION}" >&2 + # Helm mounts our chain scripts here; must NOT use /scripts — that shadows gdcc/configbaker's /scripts/bootstrap.sh. CHAIN_SCRIPTS="${BOOTSTRAP_CHAIN_SCRIPT_DIR:-/bootstrap-chain}" CONFIGBAKER_BOOTSTRAP="${CONFIGBAKER_BOOTSTRAP_SH:-/scripts/bootstrap.sh}" @@ -18,6 +22,12 @@ if [[ -z "${DATAVERSE_INTERNAL_URL}" ]]; then exit 1 fi +if [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: DATAVERSE_API_TOKEN is set from the environment (length=${#DATAVERSE_API_TOKEN})" >&2 +else + echo "k8s-bootstrap-chain: DATAVERSE_API_TOKEN is unset (first install is OK; already-bootstrapped needs a Secret — see values bootstrapJob.compose.existingAdminApiTokenSecret)" >&2 +fi + try_version() { curl -sf --max-time 15 "${DATAVERSE_INTERNAL_URL%/}/api/info/version" >/dev/null 2>&1 } @@ -55,42 +65,49 @@ if [[ "${BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP:-}" == 1 ]]; then echo "k8s-bootstrap-chain: skipping bootstrap.sh (using existing admin API token)" >&2 write_api_key_from_token "${DATAVERSE_API_TOKEN}" else - mkdir -p "$(dirname "${TOKEN_FILE}")" - if [[ -d "${TOKEN_FILE}" ]]; then - echo "k8s-bootstrap-chain: ${TOKEN_FILE} is a directory" >&2 - exit 3 - fi - umask 077 - [[ -f "${TOKEN_FILE}" ]] || : >"${TOKEN_FILE}" - - echo "k8s-bootstrap-chain: running configbaker (${CONFIGBAKER_BOOTSTRAP} -e ${TOKEN_FILE} dev) ..." >&2 - if [[ ! -x "${CONFIGBAKER_BOOTSTRAP}" ]]; then - echo "k8s-bootstrap-chain: ${CONFIGBAKER_BOOTSTRAP} missing or not executable (is /scripts shadowed by a volume mount?)" >&2 - exit 127 - fi - "${CONFIGBAKER_BOOTSTRAP}" -e "${TOKEN_FILE}" dev - - # Kubernetes may inject DATAVERSE_API_TOKEN via secretKeyRef; configbaker's bootstrap.env can still - # assign empty vars — sourcing must not wipe that token. - DATAVERSE_API_TOKEN_PRE_SOURCE="${DATAVERSE_API_TOKEN:-}" - - # shellcheck disable=SC1090 - set +e - [[ -f "${TOKEN_FILE}" ]] && source "${TOKEN_FILE}" - set -e - - if [[ -n "${API_TOKEN:-}" ]]; then - write_api_key_from_token "${API_TOKEN}" - elif [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then - echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} (configbaker often skips when already bootstrapped); using DATAVERSE_API_TOKEN from environment" >&2 + # Secret (optional ref) + non-empty token: skip configbaker — it only prints "already bootstrapped" and leaves no API_TOKEN in bootstrap.env. + if [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: skipping configbaker — using DATAVERSE_API_TOKEN from environment" >&2 write_api_key_from_token "${DATAVERSE_API_TOKEN}" - elif [[ -n "${DATAVERSE_API_TOKEN_PRE_SOURCE:-}" ]]; then - echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE}; bootstrap.env cleared DATAVERSE_API_TOKEN — using token from environment (e.g. Secret) before source" >&2 - write_api_key_from_token "${DATAVERSE_API_TOKEN_PRE_SOURCE}" else - echo "k8s-bootstrap-chain: no API_TOKEN after bootstrap and no DATAVERSE_API_TOKEN in environment." >&2 - echo "k8s-bootstrap-chain: fix: create a Secret with a superuser API token and set bootstrapJob.compose.existingAdminApiTokenSecret in Helm values (see ops/demo-deploy.tmpl.yaml), or run with BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP=1 and DATAVERSE_API_TOKEN." >&2 - exit 2 + mkdir -p "$(dirname "${TOKEN_FILE}")" + if [[ -d "${TOKEN_FILE}" ]]; then + echo "k8s-bootstrap-chain: ${TOKEN_FILE} is a directory" >&2 + exit 3 + fi + umask 077 + [[ -f "${TOKEN_FILE}" ]] || : >"${TOKEN_FILE}" + + echo "k8s-bootstrap-chain: running configbaker (${CONFIGBAKER_BOOTSTRAP} -e ${TOKEN_FILE} dev) ..." >&2 + if [[ ! -x "${CONFIGBAKER_BOOTSTRAP}" ]]; then + echo "k8s-bootstrap-chain: ${CONFIGBAKER_BOOTSTRAP} missing or not executable (is /scripts shadowed by a volume mount?)" >&2 + exit 127 + fi + "${CONFIGBAKER_BOOTSTRAP}" -e "${TOKEN_FILE}" dev + + # Kubernetes may inject DATAVERSE_API_TOKEN via secretKeyRef; configbaker's bootstrap.env can still + # assign empty vars — sourcing must not wipe that token. + DATAVERSE_API_TOKEN_PRE_SOURCE="${DATAVERSE_API_TOKEN:-}" + + # shellcheck disable=SC1090 + set +e + [[ -f "${TOKEN_FILE}" ]] && source "${TOKEN_FILE}" + set -e + + if [[ -n "${API_TOKEN:-}" ]]; then + write_api_key_from_token "${API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} (configbaker often skips when already bootstrapped); using DATAVERSE_API_TOKEN from environment" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN_PRE_SOURCE:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE}; bootstrap.env cleared DATAVERSE_API_TOKEN — using token from environment (e.g. Secret) before source" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN_PRE_SOURCE}" + else + echo "k8s-bootstrap-chain: no API_TOKEN after configbaker and no usable DATAVERSE_API_TOKEN." >&2 + echo "k8s-bootstrap-chain: fix: kubectl create secret generic ... --from-literal=token=SUPERUSER_UUID and set bootstrapJob.compose.existingAdminApiTokenSecret (see ops/demo-deploy.tmpl.yaml)." >&2 + echo "k8s-bootstrap-chain: if startup logs lack \"chain script revision=${CHAIN_SCRIPT_REVISION}\", the Helm bootstrap-chain ConfigMap is stale — helm upgrade the release or kubectl apply the rendered templates/bootstrap-chain-configmap.yaml manifest." >&2 + exit 2 + fi fi fi From c0b651ec47ff4b2809b4b9edd6d5c53996cc1e08 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 20:26:31 -0700 Subject: [PATCH 24/31] Update seeded content script, dataverse is expecting JSON booleans --- scripts/seed-content.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/seed-content.sh b/scripts/seed-content.sh index 04a02ad..60d0bb6 100755 --- a/scripts/seed-content.sh +++ b/scripts/seed-content.sh @@ -96,12 +96,26 @@ upload_file() { _dir=$3 _tab=$4 _enc=$(encode_pid "$_pid") - _json=$(printf '{"description":"DataverseUp seed","directoryLabel":"%s","restrict":"false","tabIngest":"%s"}' "$_dir" "$_tab") - curl -fsS --max-time 300 --globoff -X POST \ + # Booleans must be JSON true/false (not quoted strings); include categories per Native API examples. + _json=$(printf '{"description":"DataverseUp seed","directoryLabel":"%s","restrict":false,"tabIngest":%s,"categories":["Data"]}' "$_dir" "$_tab") + _resp=$(curl -sS --max-time 300 --globoff -X POST \ -H "X-Dataverse-key: ${TOKEN}" \ -F "file=@${_path}" \ -F "jsonData=${_json}" \ - "${API}/datasets/:persistentId/add?persistentId=${_enc}" + -w "\n%{http_code}" \ + "${API}/datasets/:persistentId/add?persistentId=${_enc}" || printf '%s\n' "000") + _code=$(printf '%s\n' "$_resp" | tail -n 1) + _body=$(printf '%s\n' "$_resp" | sed '$d') + case "$_code" in + 200|201|204) + return 0 + ;; + *) + echo "seed-content: upload $(basename "$_path") failed HTTP ${_code}" >&2 + printf '%s\n' "$_body" >&2 + exit 1 + ;; + esac } # Datasets cannot be published while their host collection is still unpublished (API often returns 403). From 167da7a80295e62faf23840ce373aaf64a4642f7 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 21:04:33 -0700 Subject: [PATCH 25/31] Amazon SDk doesnt play nice --- charts/dataverseup/values.yaml | 5 +++-- docs/HELM.md | 13 ++++++++++++- init.d/006-s3-aws-storage.sh | 21 ++++++++++++--------- ops/demo-deploy.tmpl.yaml | 3 ++- scripts/seed-content.sh | 7 +++++++ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index 3104907..a281ced 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -271,8 +271,9 @@ awsS3: # Name of a Secret you create out-of-band (keys = secretKeys below). Required when enabled. existingSecret: "" bucketName: "your-bucket-name" - endpointUrl: "https://s3.us-west-2.amazonaws.com" - # AWS SigV4 signing region (e.g. us-west-2). Must match bucket/endpoint; NOT the app name. See docs/HELM.md. + # Leave empty for real Amazon S3 (recommended). Set for MinIO / S3-compatible only (e.g. https://minio:9000). + endpointUrl: "" + # AWS SigV4 signing region (e.g. us-west-2). Must match bucket; set in mounted config + AWS_REGION on Deployment. region: "us-west-2" # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. profile: default diff --git a/docs/HELM.md b/docs/HELM.md index e8c79e9..8c066eb 100644 --- a/docs/HELM.md +++ b/docs/HELM.md @@ -137,7 +137,7 @@ Set **`initdFromChart.enabled: true`** in values to include **all** `files/init. ## S3 file storage -1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `endpointUrl`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. +1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. For **Amazon S3**, leave **`endpointUrl` empty** so Payara’s init script does not set `custom-endpoint-url` (a regional `https://s3….amazonaws.com` URL there commonly causes upload failures). Set **`endpointUrl` only** for MinIO or other S3-compatible endpoints. 2. Create a **generic** Secret in the **same namespace** as the Helm release, **before** pods that mount it start. Key names must match `awsS3.secretKeys` (defaults below): the values are the **raw file contents** of `~/.aws/credentials` and `~/.aws/config`. @@ -176,6 +176,17 @@ Set **`initdFromChart.enabled: true`** in values to include **all** `files/init. **Note:** The Java AWS SDK inside the app may not perform the same **assume-role chaining** as the AWS CLI from a complex `config` file. Prefer putting **direct** user keys or **already-assumed** temporary keys in the Secret for the app, or use EKS **IRSA** (service account + role) instead of long-lived keys if your platform supports it. +### Troubleshooting: `Failed to save the content of the uploaded file` + +The Native API returns **HTTP 400** with that message when the request reached Dataverse but **writing to the configured store failed**. This is not a bug in the seed script’s `jsonData` shape. + +With **`dataverse.files.storage-driver-id=S3`** (see `init.d/006-s3-aws-storage.sh`): + +1. **IAM** — The principal in your `aws-s3-credentials` Secret needs at least **`s3:PutObject`**, **`s3:GetObject`**, **`s3:DeleteObject`**, and **`s3:ListBucket`** on the target bucket (and prefixes Dataverse uses). Missing `PutObject` often surfaces exactly as this generic message; the real error is in server logs. +2. **Bucket and region** — `awsS3.bucketName` and `awsS3.region` must match the bucket. For **Amazon S3**, keep **`awsS3.endpointUrl` empty**; do not point it at `https://s3..amazonaws.com` unless you are on a non-AWS S3-compatible store. +3. **Credentials mounted** — After creating or rotating the Secret, **restart** Dataverse pods so `AWS_SHARED_CREDENTIALS_FILE` / `AWS_CONFIG_FILE` point at the new files. +4. **Logs** — Check the Dataverse Deployment pod logs for nested exceptions (`AccessDenied`, `NoSuchBucket`, SSL, etc.). + ## Upgrades - Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). diff --git a/init.d/006-s3-aws-storage.sh b/init.d/006-s3-aws-storage.sh index beee273..2e2afa0 100644 --- a/init.d/006-s3-aws-storage.sh +++ b/init.d/006-s3-aws-storage.sh @@ -37,14 +37,10 @@ if [ -n "${aws_bucket_name:-}" ]; then | grep -v -E '^create-system-properties dataverse\.files\.S3\.' \ | grep -v -E '^deploy ' > "$_pb_pre" || true grep -E '^deploy ' "$POSTBOOT_COMMANDS_FILE" > "$_pb_dep" || true - _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') - # With custom-endpoint-url set, Dataverse's S3AccessIO uses JVM key custom-endpoint-region for SigV4. - # If unset, upstream defaults to the literal string "dataverse" → S3 400 "region 'dataverse' is wrong". + # Amazon S3: leave custom-endpoint-url unset so the AWS SDK uses default resolution (virtual-hosted buckets, + # correct SigV4). Setting a regional URL here often breaks uploads with opaque "Failed to save the content" errors. + # MinIO / S3-compatible: set aws_endpoint_url (Helm awsS3.endpointUrl) to your service base URL. _s3_reg="${aws_s3_region:-${AWS_REGION:-}}" - if [ -z "${_s3_reg}" ]; then - echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when using custom-endpoint-url" >&2 - return 1 - fi { cat "$_pb_pre" echo "create-system-properties dataverse.files.S3.type=s3" @@ -55,8 +51,15 @@ if [ -n "${aws_bucket_name:-}" ]; then echo "create-system-properties dataverse.files.S3.connection-pool-size=4096" echo "create-system-properties dataverse.files.storage-driver-id=S3" echo "create-system-properties dataverse.files.S3.profile=${aws_s3_profile}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" + if [ -n "${aws_endpoint_url:-}" ]; then + if [ -z "${_s3_reg}" ]; then + echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when aws_endpoint_url is set" >&2 + return 1 + fi + _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') + echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" + echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" + fi cat "$_pb_dep" } > "$POSTBOOT_COMMANDS_FILE" trap - EXIT diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index ab95935..8d0247c 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -50,7 +50,8 @@ awsS3: # Kubernetes Secret you create out-of-band (see comments above). existingSecret: "aws-s3-credentials" bucketName: "demo-dataverseup" - endpointUrl: "https://s3.us-west-2.amazonaws.com" + # Empty for Amazon S3 — do not set a regional s3.*.amazonaws.com URL here; it breaks SDK uploads. Use endpointUrl only for MinIO / compatible stores. + endpointUrl: "" region: us-west-2 # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. profile: default diff --git a/scripts/seed-content.sh b/scripts/seed-content.sh index 60d0bb6..cc93300 100755 --- a/scripts/seed-content.sh +++ b/scripts/seed-content.sh @@ -113,6 +113,13 @@ upload_file() { *) echo "seed-content: upload $(basename "$_path") failed HTTP ${_code}" >&2 printf '%s\n' "$_body" >&2 + case "$_body" in + *Failed*to*save*the*content*) + echo "seed-content: hint: Dataverse accepted the upload but could not write to file storage." >&2 + echo "seed-content: hint: With awsS3.enabled, check IAM (s3:PutObject/GetObject/DeleteObject/ListBucket on the bucket), bucket name/region vs values, and that pods were restarted after creating aws-s3-credentials. See docs/HELM.md (S3 troubleshooting)." >&2 + echo "seed-content: hint: Inspect the Dataverse pod logs for AWS SDK errors (AccessDenied, NoSuchBucket, etc.)." >&2 + ;; + esac exit 1 ;; esac From ce4591e2257bd7206516f2377178b593db844002 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 22:38:22 -0700 Subject: [PATCH 26/31] Update documentatin --- .gitignore | 3 +-- README.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3016ddb..5b479ff 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,4 @@ minio-data/ data/ /docroot/ *.war -.idea/ -test-k3d-deploy \ No newline at end of file +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index f606301..90d2bc1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DataverseUp -Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and room to grow toward hosted AWS/Kubernetes without forking core. +Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and a **Helm chart** for Kubernetes without forking core. ## Quick start (local / lab) @@ -46,6 +46,14 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G ``` If you need a manual token, put it on one line in **`secrets/api/key`** (superuser token from the UI), then run **`dev_branding`** again. +## Kubernetes (Helm) + +- **Chart:** `charts/dataverseup` — see **[charts/dataverseup/README.md](charts/dataverseup/README.md)** for a feature summary and `helm` commands. +- **Runbook-style notes:** **[docs/HELM.md](docs/HELM.md)** (prereqs, Secrets, optional internal Solr, S3, bootstrap modes, smoke checks). +- **Install helper:** from the repo root, `./bin/helm_deploy RELEASE_NAME NAMESPACE` (optional `HELM_EXTRA_ARGS` for values files and timeouts). Same details are in `docs/HELM.md`. + +Compose remains the default path for local/lab; Helm reuses the same **`init.d/`** scripts (via chart symlinks under `charts/dataverseup/files/`) where applicable. + ## Layout | Path | Purpose | @@ -61,6 +69,9 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G | `fixtures/seed/` | JSON + files for **`dev_seed`** | | `scripts/` | Bootstrap, branding, seed entrypoints, `apply-branding.sh`, `solr-initdb/` | | `triggers/` | Postgres notify + optional webhook script (see **`WEBHOOK`** in `.env.example`) | +| `charts/dataverseup/` | Helm chart for Dataverse on Kubernetes (optional Solr, bootstrap Job, S3, Ingress, …) | +| `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/HELM.md`**) | +| `docs/HELM.md` | Helm install notes, values, and operational gotchas | | `docs/DEPLOYMENT.md` | **Working deployment notes + learnings** (add in-repo when you maintain runbooks) | ## Version pin From 41787389e74c03c66cf02550125812727d14e7a1 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 22:43:35 -0700 Subject: [PATCH 27/31] dry the code and refactor t condese docs --- README.md | 8 +- charts/dataverseup/README.md | 6 +- .../dataverseup/files/006-s3-aws-storage.sh | 1 - charts/dataverseup/files/010-mailrelay-set.sh | 1 - charts/dataverseup/templates/configmap.yaml | 4 +- .../templates/solr-init-configmap.yaml | 4 +- charts/dataverseup/values.yaml | 2 +- docs/DEPLOYMENT.md | 214 +++++++++++++++++- docs/HELM.md | 210 ----------------- ops/demo-deploy.tmpl.yaml | 4 +- scripts/seed-content.sh | 2 +- 11 files changed, 223 insertions(+), 233 deletions(-) delete mode 120000 charts/dataverseup/files/006-s3-aws-storage.sh delete mode 120000 charts/dataverseup/files/010-mailrelay-set.sh delete mode 100644 docs/HELM.md diff --git a/README.md b/README.md index 90d2bc1..e219a3a 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,7 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G ## Kubernetes (Helm) - **Chart:** `charts/dataverseup` — see **[charts/dataverseup/README.md](charts/dataverseup/README.md)** for a feature summary and `helm` commands. -- **Runbook-style notes:** **[docs/HELM.md](docs/HELM.md)** (prereqs, Secrets, optional internal Solr, S3, bootstrap modes, smoke checks). -- **Install helper:** from the repo root, `./bin/helm_deploy RELEASE_NAME NAMESPACE` (optional `HELM_EXTRA_ARGS` for values files and timeouts). Same details are in `docs/HELM.md`. +- **Runbook:** **[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md)** — Helm (prereqs, Secrets, Solr, S3, bootstrap), GitHub Actions deploy, `./bin/helm_deploy`, smoke tests, upgrades. Compose remains the default path for local/lab; Helm reuses the same **`init.d/`** scripts (via chart symlinks under `charts/dataverseup/files/`) where applicable. @@ -70,9 +69,8 @@ Compose remains the default path for local/lab; Helm reuses the same **`init.d/` | `scripts/` | Bootstrap, branding, seed entrypoints, `apply-branding.sh`, `solr-initdb/` | | `triggers/` | Postgres notify + optional webhook script (see **`WEBHOOK`** in `.env.example`) | | `charts/dataverseup/` | Helm chart for Dataverse on Kubernetes (optional Solr, bootstrap Job, S3, Ingress, …) | -| `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/HELM.md`**) | -| `docs/HELM.md` | Helm install notes, values, and operational gotchas | -| `docs/DEPLOYMENT.md` | **Working deployment notes + learnings** (add in-repo when you maintain runbooks) | +| `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/DEPLOYMENT.md`**) | +| `docs/DEPLOYMENT.md` | **Deployment runbook:** Helm/Kubernetes, optional GitHub Actions, Compose pointer, learnings log | ## Version pin diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md index 077c1cd..abeb3df 100644 --- a/charts/dataverseup/README.md +++ b/charts/dataverseup/README.md @@ -18,11 +18,11 @@ helm upgrade --install release-name charts/dataverseup -n your-namespace -f your ## Documentation -See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, Secret layout, and smoke tests. +See **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)** in this repository for prerequisites, Secret layout, smoke tests, and CI deploy notes. ## Payara init scripts (S3, mail) -`files/006-s3-aws-storage.sh` and `files/010-mailrelay-set.sh` are **symbolic links** to the same scripts in the repository root **`init.d/`** (used by Docker Compose). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. +S3 and mail relay scripts live only under **`files/init.d/`** as **symbolic links** to the repository root **`init.d/`** (same scripts Docker Compose mounts). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. ## Configuration @@ -35,4 +35,4 @@ See **[docs/HELM.md](../../docs/HELM.md)** in this repository for prerequisites, | `bootstrapJob` | First-time `configbaker` bootstrap | | `ingress` | HTTP routing to Service port 80 | -Solr alignment with Docker Compose (IQSS **`config/`** files, core naming, `solrInit` overrides) is documented in **[docs/HELM.md](../../docs/HELM.md)**. +Solr alignment with Docker Compose (IQSS **`config/`** files, core naming, `solrInit` overrides) is documented in **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)**. diff --git a/charts/dataverseup/files/006-s3-aws-storage.sh b/charts/dataverseup/files/006-s3-aws-storage.sh deleted file mode 120000 index e023f77..0000000 --- a/charts/dataverseup/files/006-s3-aws-storage.sh +++ /dev/null @@ -1 +0,0 @@ -../../../init.d/006-s3-aws-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/010-mailrelay-set.sh b/charts/dataverseup/files/010-mailrelay-set.sh deleted file mode 120000 index a708548..0000000 --- a/charts/dataverseup/files/010-mailrelay-set.sh +++ /dev/null @@ -1 +0,0 @@ -../../../init.d/010-mailrelay-set.sh \ No newline at end of file diff --git a/charts/dataverseup/templates/configmap.yaml b/charts/dataverseup/templates/configmap.yaml index 51a6d11..375deb2 100644 --- a/charts/dataverseup/templates/configmap.yaml +++ b/charts/dataverseup/templates/configmap.yaml @@ -19,12 +19,12 @@ data: {{- toYaml .Values.configMap.data | nindent 2 }} {{- end }} {{- if .Values.mail.enabled }} - {{- $mailScript := .Files.Get "files/010-mailrelay-set.sh" | trim }} + {{- $mailScript := .Files.Get "files/init.d/010-mailrelay-set.sh" | trim }} 010-mailrelay-set.sh: | {{- $mailScript | nindent 4 }} {{- end }} {{- if .Values.awsS3.enabled }} - {{- $s3Script := .Files.Get "files/006-s3-aws-storage.sh" | trim }} + {{- $s3Script := .Files.Get "files/init.d/006-s3-aws-storage.sh" | trim }} 006-s3-aws-storage.sh: | {{- $s3Script | nindent 4 }} {{- end }} diff --git a/charts/dataverseup/templates/solr-init-configmap.yaml b/charts/dataverseup/templates/solr-init-configmap.yaml index a702fb6..d5683b1 100644 --- a/charts/dataverseup/templates/solr-init-configmap.yaml +++ b/charts/dataverseup/templates/solr-init-configmap.yaml @@ -75,7 +75,7 @@ data: POD_NS="" if [[ -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then POD_NS=$(tr -d '\n' < /var/run/secrets/kubernetes.io/serviceaccount/namespace) - echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory for your Dataverse version (see docs/HELM.md)." >&2 + echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory for your Dataverse version (see docs/DEPLOYMENT.md)." >&2 else echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory (namespace in kubectl context)." >&2 fi @@ -88,7 +88,7 @@ data: UPLOAD_DIR="${STAGING}/conf" else echo "solr-init: missing solrconfig.xml — ConfigMap must be the full Dataverse Solr conf directory, not schema.xml only." >&2 - echo "solr-init: See https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr and docs/HELM.md" >&2 + echo "solr-init: See https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr and docs/DEPLOYMENT.md" >&2 echo "solr-init: Example: package conf dir as solr-conf.tgz ConfigMap keys (schema.xml, solrconfig.xml, lang/, …)." >&2 echo "solr-init: Staged files under ${STAGING}:" >&2 find "${STAGING}" -type f 2>/dev/null | sort >&2 || ls -laR "${STAGING}" >&2 || true diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index a281ced..0013ca7 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -265,7 +265,7 @@ solrInit: resources: {} # AWS S3 file storage: Secret mount + env (aws_bucket_name, …) plus bundled 006-s3-aws-storage.sh mounted at -# /opt/payara/scripts/init.d/ (base image entrypoint runs that path before Payara starts). See docs/HELM.md. +# /opt/payara/scripts/init.d/ (base image entrypoint runs that path before Payara starts). See docs/DEPLOYMENT.md. awsS3: enabled: false # Name of a Secret you create out-of-band (keys = secretKeys below). Required when enabled. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 0979e9e..625fdb7 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,19 +1,141 @@ -# Deployment notes (working document) +# Deployment (working document) -Rough notes for standing up Dataverse for Notch8. Extend into a full runbook as you validate each environment. +Rough notes for standing up Dataverse for Notch8: **Docker Compose** (lab), **Kubernetes / Helm**, optional **GitHub Actions** deploys, and a shared **learnings** table. Extend into a full runbook as you validate each environment. ## Ticket context (internal) - **Target:** Dataverse **v6.10** on **AWS** by **April 7, 2026** — functional demo, not necessarily production-hardened. -- **Deliverable:** Working deployment **and documented process + learnings** (this file, plus **[HELM.md](HELM.md)** for Kubernetes). +- **Deliverable:** Working deployment **and documented process + learnings** (this document). ## Docker Compose (local / lab) See repository **[README.md](../README.md)** — `docker compose up` after `.env` and `secrets/` from examples. -## Kubernetes / Helm +--- -See **[HELM.md](HELM.md)** for chart path, **`bin/helm_deploy`**, Secret layout, Solr ConfigMap, and smoke tests. +## Kubernetes: Helm chart + +The following sections describe how to install the **`dataverseup`** Helm chart from this repository. + +**Prerequisites:** Helm 3, a Kubernetes cluster, `kubectl` configured, a **PostgreSQL** database reachable from the cluster (in-cluster or managed), and a **StorageClass** for any PVCs you enable. + +**Chart path:** `charts/dataverseup` + +### `bin/helm_deploy` (recommended wrapper) + +From the **repository root**, installs or upgrades the chart with **`--install`**, **`--atomic`**, **`--create-namespace`**, and a default **`--timeout 30m0s`** (Payara first boot is slow). + +```text +./bin/helm_deploy RELEASE_NAME NAMESPACE +``` + +Pass extra Helm flags with **`HELM_EXTRA_ARGS`** (values file, longer timeout, etc.). If you pass a **second `--timeout`** in `HELM_EXTRA_ARGS`, it overrides the default (Helm uses the last value). + +```bash +HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace +``` + +### What the chart deploys + +- **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. +- **Optional bootstrap Job** (`gdcc/configbaker`) — usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). **`bootstrapJob.mode: oneShot`** runs **`bootstrapJob.command`** only (default: `bootstrap.sh dev` — FAKE DOI, `dataverseAdmin`, etc.). **`bootstrapJob.mode: compose`** mirrors local Docker Compose: wait for the API, run configbaker with a writable token file on `emptyDir`, then **`apply-branding.sh`** and **`seed-content.sh`** (fixtures baked into a ConfigMap). Tune waits with **`bootstrapJob.compose`** and allow a longer **`bootstrapJob.timeout`** when seeding. +- **Optional dedicated Solr** (`internalSolr`) — a **new** Solr Deployment/Service in the **same release and namespace** as Dataverse (not wiring into someone else’s shared “cluster Solr”). Default **`solrInit.mode`** is **`standalone`**: the Dataverse pod waits for that Solr core before starting. Use **`solrInit.mode: cloud`** only when Dataverse talks to **SolrCloud + ZooKeeper** you operate separately. +- **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. + +#### Branding (navbar logo + Admin API settings) + +1. **Navbar SVG** — Enable **`brandingNavbarLogos.enabled`** so an init container copies **`branding/docroot/logos/navbar/logo.svg`** from the chart onto **`/dv/docroot/logos/navbar/logo.svg`** (needs **`docrootPersistence`** or the chart’s emptyDir docroot fallback). Match **`LOGO_CUSTOMIZATION_FILE`** in **`branding/branding.env`** to the web path (e.g. `/logos/navbar/logo.svg`). + +2. **Admin settings** (installation name, footer, optional custom header/footer CSS paths) — Edit **`branding/branding.env`** in the repo. The chart embeds it in the **`…-bootstrap-chain`** ConfigMap when **`bootstrapJob.mode: compose`**. The post-install Job runs **`apply-branding.sh`**, which PUTs those settings via the Dataverse Admin API using the admin token from configbaker. + +3. **Custom HTML/CSS files** — Add them under **`branding/docroot/branding/`** in the repo, set **`HEADER_CUSTOMIZATION_FILE`**, etc. in **`branding.env`** to **`/dv/docroot/branding/...`**, and ship those files into the pod (extra **`volumeMounts`** / **`configMap`** or bake into an image). The stock chart does not mount the whole **`branding/docroot/branding/`** tree on the main Deployment; compose only ships **`branding.env`** and the logo via **`brandingNavbarLogos`**. + +4. **After `helm upgrade`** — The post-install hook does **not** re-run. To re-apply branding, use **`bootstrapJob.compose.postUpgradeBrandingSeedJob`** with a Secret holding **`DATAVERSE_API_TOKEN`**, or run **`scripts/apply-branding.sh`** locally/cron with **`DATAVERSE_INTERNAL_URL`** and a token. + +The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). + +#### Recommended Solr layout: new instance with this deploy + +Enable **`internalSolr.enabled`**, **`solrInit.enabled`**, keep **`solrInit.mode: standalone`**, and supply **`solrInit.confConfigMap`**. Leave **`solrInit.solrHttpBase` empty** — the chart sets the Solr admin URL to the in-release Service (`http://-solr..svc.cluster.local:8983`). Point your app Secret at that same host/port and core (see table below). You do **not** need an existing Solr installation in the cluster. + +### Docker Compose vs Helm (Solr) + +Local **`docker-compose.yml`** and this chart both target **official Solr 9** (`solr:9.10.1`) and IQSS **`conf/solr`** files vendored under repo **`config/`** (refresh from IQSS `develop` or a release tag as in the root **`README.md`**). + +| | Docker Compose | Helm (`internalSolr` + `solrInit`) | +|---|----------------|-----------------------------------| +| Solr image pin | `solr:9.10.1` | `internalSolr.image` / `solrInit.image` default `solr:9.10.1` | +| Default core name | **`collection1`** (see `scripts/solr-initdb/01-ensure-core.sh`) | **`dataverse`** (`solr-precreate` in `internal-solr-deployment.yaml`) | +| App Solr address | `SOLR_LOCATION=solr:8983` (host:port) | With **`internalSolr.enabled`**, the chart sets **`DATAVERSE_SOLR_HOST`**, **`DATAVERSE_SOLR_PORT`**, **`DATAVERSE_SOLR_CORE`**, **`SOLR_SERVICE_*`**, and **`SOLR_LOCATION`** to the in-release Solr Service and **`solrInit.collection`** (default **`dataverse`**). The GDCC `ct` profile otherwise defaults to host **`solr`** and core **`collection1`**, which breaks Kubernetes installs if unset. | + +Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core after precreate. **SolrCloud** (`solrInit.mode: cloud`) still needs a **full** conf tree or **`solr-conf.tgz`** (including `lang/`, `stopwords.txt`, etc.) for `solr zk upconfig` — see [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +#### `solrInit` image: standalone (default) vs SolrCloud + +- **Standalone** (default, with **`internalSolr`**): the initContainer **waits** for `/solr//admin/ping` via `curl`; the default **`solr:9.10.1`** image is sufficient. This matches launching a **solo Solr** with the chart instead of consuming a shared cluster Solr Service. +- **Cloud / ZooKeeper** (optional): set **`solrInit.mode: cloud`** and **`solrInit.zkConnect`** when Dataverse uses **SolrCloud** you run elsewhere. The same container runs **`solr zk upconfig`**; use a Solr **major** compatible with that cluster. Override **`solrInit.image`**, **`solrInit.solrBin`**, and **`solrInit.securityContext`** if you use a vendor image (e.g. legacy Bitnami). + +### Install flow (recommended order) + +1. **Create namespace** + `kubectl create namespace ` + +2. **Database** + Provision Postgres and a database/user for Dataverse. Note the service DNS name inside the cluster (e.g. `postgres..svc.cluster.local`). + +3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) + Dataverse needs a **full** Solr configuration directory for its version — not `schema.xml` alone. Build a ConfigMap whose keys are the files under that conf directory (or a single `solr-conf.tgz` as produced by your packaging process). See [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +4. **Application Secret** (example name `dataverse-app-env`) + Prefer `stringData` for passwords. Include at least the variables the GDCC image expects for JDBC and Solr (mirror what you use in Docker Compose `.env`). Typical keys include: + + - `DATAVERSE_DB_HOST`, `DATAVERSE_DB_USER`, `DATAVERSE_DB_PASSWORD`, `DATAVERSE_DB_NAME` + - `POSTGRES_SERVER`, `POSTGRES_PORT`, `POSTGRES_DATABASE`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `PGPASSWORD` + - Solr: `SOLR_LOCATION` or `DATAVERSE_SOLR_HOST` / `DATAVERSE_SOLR_PORT` / `DATAVERSE_SOLR_CORE` (match your Solr deployment) + - Public URL / hostname: `DATAVERSE_URL`, `hostname`, `DATAVERSE_SERVICE_HOST` (used by init scripts and UI) + - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) + +5. **Values file** + Start from `charts/dataverseup/values.yaml` and override with a small values file of your own. At minimum for a first install: + + - `persistence.enabled: true` (file store) + - `extraEnvFrom` pointing at your Secret + - If using dedicated in-chart Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.confConfigMap`, `solrInit.mode: standalone` (default). Omit `solrInit.solrHttpBase` to use the auto-derived in-release Solr Service URL + - `bootstrapJob.enabled: true` for first-time seeding + +6. **Lint and render** + + ```bash + helm lint charts/dataverseup -f your-values.yaml + helm template dataverseup charts/dataverseup -f your-values.yaml > /tmp/manifests.yaml + ``` + +7. **Install** + + Using the wrapper (from repo root): + + ```bash + HELM_EXTRA_ARGS="--values ./your-values.yaml --wait" ./bin/helm_deploy + ``` + + Raw Helm (equivalent shape): + + ```bash + helm upgrade --install charts/dataverseup -n -f your-values.yaml --wait --timeout 45m + ``` + +8. **Smoke tests** + + - `kubectl get pods -n ` + - Bootstrap job logs (if enabled): `kubectl logs -n job/...-bootstrap` + - API: port-forward or Ingress → `GET /api/info/version` should return **200** + - UI login (default bootstrap admin from configbaker **dev** profile — **change** before any shared environment) + +9. **Helm test** (optional) + + ```bash + helm test -n + ``` ### GitHub Actions — Deploy workflow @@ -35,8 +157,90 @@ Default Helm **release** and **namespace** are **`--deploy.tmpl.yaml`** (ingress hosts, `dataverse_*` / `DATAVERSE_*` / `hostname`, `solrHttpBase`, `SOLR_*`, `DATAVERSE_URL`, `awsS3.bucketName`, DB names, etc.) so they match the new Helm release and namespace; then align Postgres, S3, TLS, and running workloads. +### Ingress and TLS + +Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. + +If you terminate TLS or expose the app on a **non-default host port**, keep **`DATAVERSE_URL`** and related hostname settings aligned with the URL users and the app use. + +### Payara init scripts (DRY with Compose) + +The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. + +Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting `./init.d`). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and triggers-specific scripts before enabling in a cluster that does not mount those paths. + +### S3 file storage + +1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. For **Amazon S3**, leave **`endpointUrl` empty** so Payara’s init script does not set `custom-endpoint-url` (a regional `https://s3….amazonaws.com` URL there commonly causes upload failures). Set **`endpointUrl` only** for MinIO or other S3-compatible endpoints. + +2. Create a **generic** Secret in the **same namespace** as the Helm release, **before** pods that mount it start. Key names must match `awsS3.secretKeys` (defaults below): the values are the **raw file contents** of `~/.aws/credentials` and `~/.aws/config`. + + - `credentials` — ini format; the profile block header (e.g. `[default]` or `[my-profile]`) must match **`awsS3.profile`**. + - `config` — ini format; for `profile: default` use `[default]` with `region = ...`. For a named profile use `[profile my-profile]` and the same region as `awsS3.region` unless you know you need otherwise. + +3. **Examples** (replace `NAMESPACE`, keys, region, and secret name if you changed `existingSecret`): + + ```sh + NS=NAMESPACE + kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - + + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-file=credentials="$HOME/.aws/credentials" \ + --from-file=config="$HOME/.aws/config" \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + Inline `[default]` user (no local files): + + ```sh + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-literal=credentials="[default] + aws_access_key_id = AKIA... + aws_secret_access_key = ... + " \ + --from-literal=config="[default] + region = us-west-2 + " \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + If you use **temporary credentials** (assumed role / STS), add a line to the credentials profile: `aws_session_token = ...`. Rotate before expiry or automate renewal. + +4. After creating or updating the Secret, **restart** the Dataverse Deployment (or delete its pods) so the volume is remounted. The chart sets `AWS_SHARED_CREDENTIALS_FILE` and `AWS_CONFIG_FILE` to the mounted paths. + +**Note:** The Java AWS SDK inside the app may not perform the same **assume-role chaining** as the AWS CLI from a complex `config` file. Prefer putting **direct** user keys or **already-assumed** temporary keys in the Secret for the app, or use EKS **IRSA** (service account + role) instead of long-lived keys if your platform supports it. + +#### Troubleshooting: `Failed to save the content of the uploaded file` + +The Native API returns **HTTP 400** with that message when the request reached Dataverse but **writing to the configured store failed**. This is not a bug in the seed script’s `jsonData` shape. + +With **`dataverse.files.storage-driver-id=S3`** (see `init.d/006-s3-aws-storage.sh`): + +1. **IAM** — The principal in your `aws-s3-credentials` Secret needs at least **`s3:PutObject`**, **`s3:GetObject`**, **`s3:DeleteObject`**, and **`s3:ListBucket`** on the target bucket (and prefixes Dataverse uses). Missing `PutObject` often surfaces exactly as this generic message; the real error is in server logs. +2. **Bucket and region** — `awsS3.bucketName` and `awsS3.region` must match the bucket. For **Amazon S3**, keep **`awsS3.endpointUrl` empty**; do not point it at `https://s3..amazonaws.com` unless you are on a non-AWS S3-compatible store. +3. **Credentials mounted** — After creating or rotating the Secret, **restart** Dataverse pods so `AWS_SHARED_CREDENTIALS_FILE` / `AWS_CONFIG_FILE` point at the new files. +4. **Logs** — Check the Dataverse Deployment pod logs for nested exceptions (`AccessDenied`, `NoSuchBucket`, SSL, etc.). + +### Upgrades + +- Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). +- Reconcile Solr conf ConfigMap when Solr schema changes. +- When upgrading **internal Solr** across a **major Solr version** (e.g. 8 → 9), use a **fresh** Solr data volume (new PVC or wipe `internalSolr` persistence) so cores are recreated; same idea as Compose (see root **`README.md`**). +- After bumping **`solrInit`** / **`internalSolr`** images, re-test **SolrCloud** installs (`solr zk` + collection create) in a non-production cluster if you use `solrInit.mode: cloud`. +- If `bootstrapJob.helmHook` is **true**, the bootstrap Job runs on **post-install only**, not on every upgrade (by design). + +--- + ## Learnings log +Append rows as you go (Compose, cluster, CI, etc.): + | Date | Environment | Note | |------|-------------|------| | | | | + +## References + +- [Running Dataverse in Docker](https://guides.dataverse.org/en/latest/container/running/index.html) (conceptual parity with container env) +- [Application image](https://guides.dataverse.org/en/latest/container/app-image.html) +- [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr) diff --git a/docs/HELM.md b/docs/HELM.md deleted file mode 100644 index 8c066eb..0000000 --- a/docs/HELM.md +++ /dev/null @@ -1,210 +0,0 @@ -# Helm deployment (DataverseUp chart) - -This document describes how to install the **`dataverseup`** Helm chart from this repository. It is written as **working notes** you can extend into a full runbook after the first successful deploy. - -**Prerequisites:** Helm 3, a Kubernetes cluster, `kubectl` configured, a **PostgreSQL** database reachable from the cluster (in-cluster or managed), and a **StorageClass** for any PVCs you enable. - -**Chart path:** `charts/dataverseup` - -## `bin/helm_deploy` (recommended wrapper) - -From the **repository root**, installs or upgrades the chart with **`--install`**, **`--atomic`**, **`--create-namespace`**, and a default **`--timeout 30m0s`** (Payara first boot is slow). - -```text -./bin/helm_deploy RELEASE_NAME NAMESPACE -``` - -Pass extra Helm flags with **`HELM_EXTRA_ARGS`** (values file, longer timeout, etc.). If you pass a **second `--timeout`** in `HELM_EXTRA_ARGS`, it overrides the default (Helm uses the last value). - -```bash -HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace -``` - - -## What the chart deploys - -- **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. -- **Optional bootstrap Job** (`gdcc/configbaker`) — usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). **`bootstrapJob.mode: oneShot`** runs **`bootstrapJob.command`** only (default: `bootstrap.sh dev` — FAKE DOI, `dataverseAdmin`, etc.). **`bootstrapJob.mode: compose`** mirrors local Docker Compose: wait for the API, run configbaker with a writable token file on `emptyDir`, then **`apply-branding.sh`** and **`seed-content.sh`** (fixtures baked into a ConfigMap). Tune waits with **`bootstrapJob.compose`** and allow a longer **`bootstrapJob.timeout`** when seeding. -- **Optional dedicated Solr** (`internalSolr`) — a **new** Solr Deployment/Service in the **same release and namespace** as Dataverse (not wiring into someone else’s shared “cluster Solr”). Default **`solrInit.mode`** is **`standalone`**: the Dataverse pod waits for that Solr core before starting. Use **`solrInit.mode: cloud`** only when Dataverse talks to **SolrCloud + ZooKeeper** you operate separately. -- **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. - -### Branding (navbar logo + Admin API settings) - -1. **Navbar SVG** — Enable **`brandingNavbarLogos.enabled`** so an init container copies **`branding/docroot/logos/navbar/logo.svg`** from the chart onto **`/dv/docroot/logos/navbar/logo.svg`** (needs **`docrootPersistence`** or the chart’s emptyDir docroot fallback). Match **`LOGO_CUSTOMIZATION_FILE`** in **`branding/branding.env`** to the web path (e.g. `/logos/navbar/logo.svg`). - -2. **Admin settings** (installation name, footer, optional custom header/footer CSS paths) — Edit **`branding/branding.env`** in the repo. The chart embeds it in the **`…-bootstrap-chain`** ConfigMap when **`bootstrapJob.mode: compose`**. The post-install Job runs **`apply-branding.sh`**, which PUTs those settings via the Dataverse Admin API using the admin token from configbaker. - -3. **Custom HTML/CSS files** — Add them under **`branding/docroot/branding/`** in the repo, set **`HEADER_CUSTOMIZATION_FILE`**, etc. in **`branding.env`** to **`/dv/docroot/branding/...`**, and ship those files into the pod (extra **`volumeMounts`** / **`configMap`** or bake into an image). The stock chart does not mount the whole **`branding/docroot/branding/`** tree on the main Deployment; compose only ships **`branding.env`** and the logo via **`brandingNavbarLogos`**. - -4. **After `helm upgrade`** — The post-install hook does **not** re-run. To re-apply branding, use **`bootstrapJob.compose.postUpgradeBrandingSeedJob`** with a Secret holding **`DATAVERSE_API_TOKEN`**, or run **`scripts/apply-branding.sh`** locally/cron with **`DATAVERSE_INTERNAL_URL`** and a token. - -The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). - -### Recommended Solr layout: new instance with this deploy - -Enable **`internalSolr.enabled`**, **`solrInit.enabled`**, keep **`solrInit.mode: standalone`**, and supply **`solrInit.confConfigMap`**. Leave **`solrInit.solrHttpBase` empty** — the chart sets the Solr admin URL to the in-release Service (`http://-solr..svc.cluster.local:8983`). Point your app Secret at that same host/port and core (see table below). You do **not** need an existing Solr installation in the cluster. - -## Docker Compose vs Helm (Solr) - -Local **`docker-compose.yml`** and this chart both target **official Solr 9** (`solr:9.10.1`) and IQSS **`conf/solr`** files vendored under repo **`config/`** (refresh from IQSS `develop` or a release tag as in the root **`README.md`**). - -| | Docker Compose | Helm (`internalSolr` + `solrInit`) | -|---|----------------|-----------------------------------| -| Solr image pin | `solr:9.10.1` | `internalSolr.image` / `solrInit.image` default `solr:9.10.1` | -| Default core name | **`collection1`** (see `scripts/solr-initdb/01-ensure-core.sh`) | **`dataverse`** (`solr-precreate` in `internal-solr-deployment.yaml`) | -| App Solr address | `SOLR_LOCATION=solr:8983` (host:port) | With **`internalSolr.enabled`**, the chart sets **`DATAVERSE_SOLR_HOST`**, **`DATAVERSE_SOLR_PORT`**, **`DATAVERSE_SOLR_CORE`**, **`SOLR_SERVICE_*`**, and **`SOLR_LOCATION`** to the in-release Solr Service and **`solrInit.collection`** (default **`dataverse`**). The GDCC `ct` profile otherwise defaults to host **`solr`** and core **`collection1`**, which breaks Kubernetes installs if unset. | - -Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core after precreate. **SolrCloud** (`solrInit.mode: cloud`) still needs a **full** conf tree or **`solr-conf.tgz`** (including `lang/`, `stopwords.txt`, etc.) for `solr zk upconfig` — see [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). - -### `solrInit` image: standalone (default) vs SolrCloud - -- **Standalone** (default, with **`internalSolr`**): the initContainer **waits** for `/solr//admin/ping` via `curl`; the default **`solr:9.10.1`** image is sufficient. This matches launching a **solo Solr** with the chart instead of consuming a shared cluster Solr Service. -- **Cloud / ZooKeeper** (optional): set **`solrInit.mode: cloud`** and **`solrInit.zkConnect`** when Dataverse uses **SolrCloud** you run elsewhere. The same container runs **`solr zk upconfig`**; use a Solr **major** compatible with that cluster. Override **`solrInit.image`**, **`solrInit.solrBin`**, and **`solrInit.securityContext`** if you use a vendor image (e.g. legacy Bitnami). - -## Install flow (recommended order) - -1. **Create namespace** - `kubectl create namespace ` - -2. **Database** - Provision Postgres and a database/user for Dataverse. Note the service DNS name inside the cluster (e.g. `postgres..svc.cluster.local`). - -3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) - Dataverse needs a **full** Solr configuration directory for its version — not `schema.xml` alone. Build a ConfigMap whose keys are the files under that conf directory (or a single `solr-conf.tgz` as produced by your packaging process). See [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). - -4. **Application Secret** (example name `dataverse-app-env`) - Prefer `stringData` for passwords. Include at least the variables the GDCC image expects for JDBC and Solr (mirror what you use in Docker Compose `.env`). Typical keys include: - - - `DATAVERSE_DB_HOST`, `DATAVERSE_DB_USER`, `DATAVERSE_DB_PASSWORD`, `DATAVERSE_DB_NAME` - - `POSTGRES_SERVER`, `POSTGRES_PORT`, `POSTGRES_DATABASE`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `PGPASSWORD` - - Solr: `SOLR_LOCATION` or `DATAVERSE_SOLR_HOST` / `DATAVERSE_SOLR_PORT` / `DATAVERSE_SOLR_CORE` (match your Solr deployment) - - Public URL / hostname: `DATAVERSE_URL`, `hostname`, `DATAVERSE_SERVICE_HOST` (used by init scripts and UI) - - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) - -5. **Values file** - Start from `charts/dataverseup/values.yaml` and override with a small values file of your own. At minimum for a first install: - - - `persistence.enabled: true` (file store) - - `extraEnvFrom` pointing at your Secret - - If using dedicated in-chart Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.confConfigMap`, `solrInit.mode: standalone` (default). Omit `solrInit.solrHttpBase` to use the auto-derived in-release Solr Service URL - - `bootstrapJob.enabled: true` for first-time seeding - -6. **Lint and render** - - ```bash - helm lint charts/dataverseup -f your-values.yaml - helm template dataverseup charts/dataverseup -f your-values.yaml > /tmp/manifests.yaml - ``` - -7. **Install** - - Using the wrapper (from repo root): - - ```bash - HELM_EXTRA_ARGS="--values ./your-values.yaml --wait" ./bin/helm_deploy - ``` - - Raw Helm (equivalent shape): - - ```bash - helm upgrade --install charts/dataverseup -n -f your-values.yaml --wait --timeout 45m - ``` - -8. **Smoke tests** - - - `kubectl get pods -n ` - - Bootstrap job logs (if enabled): `kubectl logs -n job/...-bootstrap` - - API: port-forward or Ingress → `GET /api/info/version` should return **200** - - UI login (default bootstrap admin from configbaker **dev** profile — **change** before any shared environment) - -9. **Helm test** (optional) - - ```bash - helm test -n - ``` - -## Ingress and TLS - -Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. - -If you terminate TLS or expose the app on a **non-default host port**, keep **`DATAVERSE_URL`** and related hostname settings aligned with the URL users and the app use. - -## Payara init scripts (DRY with Compose) - -The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. - -Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting `./init.d`). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and triggers-specific scripts before enabling in a cluster that does not mount those paths. - -## S3 file storage - -1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. For **Amazon S3**, leave **`endpointUrl` empty** so Payara’s init script does not set `custom-endpoint-url` (a regional `https://s3….amazonaws.com` URL there commonly causes upload failures). Set **`endpointUrl` only** for MinIO or other S3-compatible endpoints. - -2. Create a **generic** Secret in the **same namespace** as the Helm release, **before** pods that mount it start. Key names must match `awsS3.secretKeys` (defaults below): the values are the **raw file contents** of `~/.aws/credentials` and `~/.aws/config`. - - - `credentials` — ini format; the profile block header (e.g. `[default]` or `[my-profile]`) must match **`awsS3.profile`**. - - `config` — ini format; for `profile: default` use `[default]` with `region = ...`. For a named profile use `[profile my-profile]` and the same region as `awsS3.region` unless you know you need otherwise. - -3. **Examples** (replace `NAMESPACE`, keys, region, and secret name if you changed `existingSecret`): - - ```sh - NS=NAMESPACE - kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - - - kubectl -n "$NS" create secret generic aws-s3-credentials \ - --from-file=credentials="$HOME/.aws/credentials" \ - --from-file=config="$HOME/.aws/config" \ - --dry-run=client -o yaml | kubectl apply -f - - ``` - - Inline `[default]` user (no local files): - - ```sh - kubectl -n "$NS" create secret generic aws-s3-credentials \ - --from-literal=credentials="[default] - aws_access_key_id = AKIA... - aws_secret_access_key = ... - " \ - --from-literal=config="[default] - region = us-west-2 - " \ - --dry-run=client -o yaml | kubectl apply -f - - ``` - - If you use **temporary credentials** (assumed role / STS), add a line to the credentials profile: `aws_session_token = ...`. Rotate before expiry or automate renewal. - -4. After creating or updating the Secret, **restart** the Dataverse Deployment (or delete its pods) so the volume is remounted. The chart sets `AWS_SHARED_CREDENTIALS_FILE` and `AWS_CONFIG_FILE` to the mounted paths. - -**Note:** The Java AWS SDK inside the app may not perform the same **assume-role chaining** as the AWS CLI from a complex `config` file. Prefer putting **direct** user keys or **already-assumed** temporary keys in the Secret for the app, or use EKS **IRSA** (service account + role) instead of long-lived keys if your platform supports it. - -### Troubleshooting: `Failed to save the content of the uploaded file` - -The Native API returns **HTTP 400** with that message when the request reached Dataverse but **writing to the configured store failed**. This is not a bug in the seed script’s `jsonData` shape. - -With **`dataverse.files.storage-driver-id=S3`** (see `init.d/006-s3-aws-storage.sh`): - -1. **IAM** — The principal in your `aws-s3-credentials` Secret needs at least **`s3:PutObject`**, **`s3:GetObject`**, **`s3:DeleteObject`**, and **`s3:ListBucket`** on the target bucket (and prefixes Dataverse uses). Missing `PutObject` often surfaces exactly as this generic message; the real error is in server logs. -2. **Bucket and region** — `awsS3.bucketName` and `awsS3.region` must match the bucket. For **Amazon S3**, keep **`awsS3.endpointUrl` empty**; do not point it at `https://s3..amazonaws.com` unless you are on a non-AWS S3-compatible store. -3. **Credentials mounted** — After creating or rotating the Secret, **restart** Dataverse pods so `AWS_SHARED_CREDENTIALS_FILE` / `AWS_CONFIG_FILE` point at the new files. -4. **Logs** — Check the Dataverse Deployment pod logs for nested exceptions (`AccessDenied`, `NoSuchBucket`, SSL, etc.). - -## Upgrades - -- Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). -- Reconcile Solr conf ConfigMap when Solr schema changes. -- When upgrading **internal Solr** across a **major Solr version** (e.g. 8 → 9), use a **fresh** Solr data volume (new PVC or wipe `internalSolr` persistence) so cores are recreated; same idea as Compose (see root **`README.md`**). -- After bumping **`solrInit`** / **`internalSolr`** images, re-test **SolrCloud** installs (`solr zk` + collection create) in a non-production cluster if you use `solrInit.mode: cloud`. -- If `bootstrapJob.helmHook` is **true**, the bootstrap Job runs on **post-install only**, not on every upgrade (by design). - -## Learnings log - -Append rows as you go (cluster type, storage class, what broke, what fixed it): - -| Date | Cluster | Note | -|------|---------|------| -| | | | - -## References - -- [Running Dataverse in Docker](https://guides.dataverse.org/en/latest/container/running/index.html) (conceptual parity with container env) -- [Application image](https://guides.dataverse.org/en/latest/container/app-image.html) -- [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr) diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 8d0247c..3dc30b3 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -11,7 +11,7 @@ # envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml # # Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), -# same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/HELM.md**). +# same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/DEPLOYMENT.md**). # # With **awsS3.enabled**, create a **generic** Secret in the release namespace **before** Helm installs pods that # mount it. The chart mounts it as AWS CLI files under `/secrets/aws-cli/.aws/`. @@ -43,7 +43,7 @@ # # C) STS / assumed-role: add `aws_session_token = ...` under the same profile in the credentials file body. # # After creating or rotating the Secret, restart the Dataverse Deployment so the volume remounts. -# Full detail: **docs/HELM.md** (S3 file storage). +# Full detail: **docs/DEPLOYMENT.md** (S3 file storage). awsS3: enabled: true diff --git a/scripts/seed-content.sh b/scripts/seed-content.sh index cc93300..c0588bc 100755 --- a/scripts/seed-content.sh +++ b/scripts/seed-content.sh @@ -116,7 +116,7 @@ upload_file() { case "$_body" in *Failed*to*save*the*content*) echo "seed-content: hint: Dataverse accepted the upload but could not write to file storage." >&2 - echo "seed-content: hint: With awsS3.enabled, check IAM (s3:PutObject/GetObject/DeleteObject/ListBucket on the bucket), bucket name/region vs values, and that pods were restarted after creating aws-s3-credentials. See docs/HELM.md (S3 troubleshooting)." >&2 + echo "seed-content: hint: With awsS3.enabled, check IAM (s3:PutObject/GetObject/DeleteObject/ListBucket on the bucket), bucket name/region vs values, and that pods were restarted after creating aws-s3-credentials. See docs/DEPLOYMENT.md (S3 troubleshooting)." >&2 echo "seed-content: hint: Inspect the Dataverse pod logs for AWS SDK errors (AccessDenied, NoSuchBucket, etc.)." >&2 ;; esac From b50b2bd454527ac6a550f792dbf1704940cd805f Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 22:51:54 -0700 Subject: [PATCH 28/31] dry the code and refactor scripts --- .env.example | 12 +-- README.md | 15 ++-- charts/dataverseup/README.md | 2 +- .../files/init.d/006-s3-aws-storage.sh | 2 +- .../files/init.d/01-persistent-id.sh | 2 +- .../dataverseup/files/init.d/010-languages.sh | 2 +- .../files/init.d/010-mailrelay-set.sh | 2 +- .../files/init.d/011-local-storage.sh | 2 +- .../files/init.d/012-minio-bucket1.sh | 2 +- .../files/init.d/013-minio-bucket2.sh | 2 +- .../files/init.d/02-controlled-voc.sh | 2 +- charts/dataverseup/files/init.d/03-doi-set.sh | 2 +- .../dataverseup/files/init.d/04-setdomain.sh | 2 +- charts/dataverseup/files/init.d/05-reindex.sh | 2 +- .../dataverseup/files/init.d/07-previewers.sh | 2 +- .../files/init.d/08-federated-login.sh | 2 +- .../dataverseup/files/init.d/1001-webhooks.sh | 2 +- .../files/init.d/1002-custom-metadata.sh | 2 +- charts/dataverseup/templates/_helpers.tpl | 77 +++++++++++++++++++ .../templates/bootstrap-chain-configmap.yaml | 2 +- .../bootstrap-compose-postupgrade-job.yaml | 50 +----------- .../dataverseup/templates/bootstrap-job.yaml | 55 +------------ charts/dataverseup/values.yaml | 2 +- docker-compose.yml | 6 +- docs/DEPLOYMENT.md | 6 +- ops/demo-deploy.tmpl.yaml | 10 +-- .../init.d}/006-s3-aws-storage.sh | 0 .../init.d}/01-persistent-id.sh | 0 {init.d => scripts/init.d}/010-languages.sh | 0 .../init.d}/010-mailrelay-set.sh | 2 +- .../init.d}/011-local-storage.sh | 0 .../init.d}/012-minio-bucket1.sh | 0 .../init.d}/013-minio-bucket2.sh | 0 .../init.d}/02-controlled-voc.sh | 0 {init.d => scripts/init.d}/03-doi-set.sh | 0 {init.d => scripts/init.d}/04-setdomain.sh | 0 {init.d => scripts/init.d}/05-reindex.sh | 0 {init.d => scripts/init.d}/07-previewers.sh | 0 .../init.d}/08-federated-login.sh | 0 {init.d => scripts/init.d}/1001-webhooks.sh | 0 .../init.d}/1002-custom-metadata.sh | 2 +- .../init.d}/vendor-solr/update-fields.sh | 0 .../init.d}/vendor-solr/updateSchemaMDB.sh | 0 {config => scripts/solr}/update-fields.sh | 0 .../triggers}/affiliations.sql | 0 .../triggers}/external-service.sql | 0 .../triggers}/external-services.py | 0 .../triggers}/lang-properties-convert.py | 0 48 files changed, 125 insertions(+), 146 deletions(-) rename {init.d => scripts/init.d}/006-s3-aws-storage.sh (100%) rename {init.d => scripts/init.d}/01-persistent-id.sh (100%) rename {init.d => scripts/init.d}/010-languages.sh (100%) rename {init.d => scripts/init.d}/010-mailrelay-set.sh (93%) rename {init.d => scripts/init.d}/011-local-storage.sh (100%) rename {init.d => scripts/init.d}/012-minio-bucket1.sh (100%) rename {init.d => scripts/init.d}/013-minio-bucket2.sh (100%) rename {init.d => scripts/init.d}/02-controlled-voc.sh (100%) rename {init.d => scripts/init.d}/03-doi-set.sh (100%) rename {init.d => scripts/init.d}/04-setdomain.sh (100%) rename {init.d => scripts/init.d}/05-reindex.sh (100%) rename {init.d => scripts/init.d}/07-previewers.sh (100%) rename {init.d => scripts/init.d}/08-federated-login.sh (100%) rename {init.d => scripts/init.d}/1001-webhooks.sh (100%) rename {init.d => scripts/init.d}/1002-custom-metadata.sh (93%) rename {init.d => scripts/init.d}/vendor-solr/update-fields.sh (100%) rename {init.d => scripts/init.d}/vendor-solr/updateSchemaMDB.sh (100%) rename {config => scripts/solr}/update-fields.sh (100%) rename {triggers => scripts/triggers}/affiliations.sql (100%) rename {triggers => scripts/triggers}/external-service.sql (100%) rename {triggers => scripts/triggers}/external-services.py (100%) rename {triggers => scripts/triggers}/lang-properties-convert.py (100%) diff --git a/.env.example b/.env.example index bc57ec4..8b29357 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ DATAVERSE_DB_NAME=dataverse SOLR_SERVICE_HOST=solr SOLR_SERVICE_PORT=8983 -# Used by init.d scripts inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme. +# Used by scripts/init.d inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme. DATAVERSE_URL=localhost:8080 DATAVERSE_SERVICE_HOST=localhost.direct # Optional: admin API token for branding/seed compose jobs (normally secrets/api/key is filled from bootstrap.env by dev_branding / dev_seed entrypoints). @@ -26,7 +26,7 @@ POSTGRES_DATABASE=dataverse POSTGRES_DB=dataverse POSTGRES_PORT=5432 -# Public hostname (init.d sets dataverse.siteUrl to https:// — use Stack Car *.localhost.direct). +# Public hostname (scripts/init.d sets dataverse.siteUrl to https:// — use Stack Car *.localhost.direct). hostname=localhost.direct # Traefik Host() rules use ${traefikhost} and subdomains (solr., minio., …). traefikhost=localhost.direct @@ -41,20 +41,20 @@ useremail=you@example.org WEBHOOK=/opt/payara/triggers/external-services.py -# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / init.d) +# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / scripts/init.d) dataciterestapiurlstring=https\\:\/\/api.test.datacite.org baseurlstring=https\:\/\/mds.test.datacite.org -# Optional builtin-user API key override (see init.d/01-persistent-id.sh) +# Optional builtin-user API key override (see scripts/init.d/01-persistent-id.sh) # BUILTIN_USERS_KEY=burrito -# --- Optional: MinIO buckets (only if init.d minio scripts should run) --- +# --- Optional: MinIO buckets (only if scripts/init.d minio scripts should run) --- # minio_label_1= # bucketname_1= # minio_bucket_1= # minio_profile_1= -# --- Optional: AWS S3 store (only if init.d/006-s3-aws-storage.sh should run) --- +# --- Optional: AWS S3 store (only if scripts/init.d/006-s3-aws-storage.sh should run) --- # aws_bucket_name= # aws_endpoint_url= # aws_s3_profile= diff --git a/README.md b/README.md index e219a3a..f23a1ff 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G - **Chart:** `charts/dataverseup` — see **[charts/dataverseup/README.md](charts/dataverseup/README.md)** for a feature summary and `helm` commands. - **Runbook:** **[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md)** — Helm (prereqs, Secrets, Solr, S3, bootstrap), GitHub Actions deploy, `./bin/helm_deploy`, smoke tests, upgrades. -Compose remains the default path for local/lab; Helm reuses the same **`init.d/`** scripts (via chart symlinks under `charts/dataverseup/files/`) where applicable. +Compose remains the default path for local/lab; Helm reuses the same Payara scripts from **`scripts/init.d/`** (chart symlinks under `charts/dataverseup/files/init.d/`) where applicable. ## Layout @@ -60,14 +60,11 @@ Compose remains the default path for local/lab; Helm reuses the same **`init.d/` | `docker-compose.yml` | Stack: Postgres, Solr, MinIO (optional), Dataverse, bootstrap, branding, seed; Traefik labels + **`networks.default.name: stackcar`** (Stack Car proxy) | | `.env.example` | Version pins and env template — copy to `.env` | | `secrets.example/` | Payara/Dataverse secret files template — copy to **`secrets/`** (see Quick start) | -| `init.d/` | Payara init scripts (local storage, optional S3/MinIO when env set) | -| `init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` | -| `config/schema.xml`, `config/solrconfig.xml` | Solr conf bind-mounts / upstream copies (see `scripts/solr-initdb/`) | -| `config/update-fields.sh` | Upstream metadata-block tooling helper | +| `scripts/` | **All automation:** Payara **`init.d/`**, Compose/K8s entrypoints, **`apply-branding.sh`**, **`seed-content.sh`**, **`solr-initdb/`**, **`solr/update-fields.sh`**, **`triggers/`**, **`k8s/`** | +| `scripts/init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` | +| `config/schema.xml`, `config/solrconfig.xml` | Solr XML bind-mounts / upstream copies (see `scripts/solr-initdb/`) | | `branding/` | Installation branding + static assets | | `fixtures/seed/` | JSON + files for **`dev_seed`** | -| `scripts/` | Bootstrap, branding, seed entrypoints, `apply-branding.sh`, `solr-initdb/` | -| `triggers/` | Postgres notify + optional webhook script (see **`WEBHOOK`** in `.env.example`) | | `charts/dataverseup/` | Helm chart for Dataverse on Kubernetes (optional Solr, bootstrap Job, S3, Ingress, …) | | `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/DEPLOYMENT.md`**) | | `docs/DEPLOYMENT.md` | **Deployment runbook:** Helm/Kubernetes, optional GitHub Actions, Compose pointer, learnings log | @@ -82,8 +79,8 @@ Compose uses **`solr:9.10.1`** with IQSS **`schema.xml`** / **`solrconfig.xml`** REF=develop # or a release tag, e.g. v6.10.1 curl -fsSL -o config/schema.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/schema.xml" curl -fsSL -o config/solrconfig.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/solrconfig.xml" -curl -fsSL -o config/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh" -chmod +x config/update-fields.sh +curl -fsSL -o scripts/solr/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh" +chmod +x scripts/solr/update-fields.sh ``` If you previously ran Solr 8, remove the compose Solr volume once so the core is recreated under Solr 9, then reindex from Dataverse. diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md index abeb3df..701d87c 100644 --- a/charts/dataverseup/README.md +++ b/charts/dataverseup/README.md @@ -22,7 +22,7 @@ See **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)** in this repository for pr ## Payara init scripts (S3, mail) -S3 and mail relay scripts live only under **`files/init.d/`** as **symbolic links** to the repository root **`init.d/`** (same scripts Docker Compose mounts). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. +S3 and mail relay scripts (and the full Payara bundle when `initdFromChart.enabled`) live under **`files/init.d/`** as **symbolic links** to **`scripts/init.d/`** (same tree Docker Compose mounts at `/opt/payara/init.d`). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. ## Configuration diff --git a/charts/dataverseup/files/init.d/006-s3-aws-storage.sh b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh index 9c9b8ab..8ef49dd 120000 --- a/charts/dataverseup/files/init.d/006-s3-aws-storage.sh +++ b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh @@ -1 +1 @@ -../../../../init.d/006-s3-aws-storage.sh \ No newline at end of file +../../../../scripts/init.d/006-s3-aws-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/01-persistent-id.sh b/charts/dataverseup/files/init.d/01-persistent-id.sh index 0d76aa3..8c436eb 120000 --- a/charts/dataverseup/files/init.d/01-persistent-id.sh +++ b/charts/dataverseup/files/init.d/01-persistent-id.sh @@ -1 +1 @@ -../../../../init.d/01-persistent-id.sh \ No newline at end of file +../../../../scripts/init.d/01-persistent-id.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-languages.sh b/charts/dataverseup/files/init.d/010-languages.sh index a71e6e7..b4b5fbb 120000 --- a/charts/dataverseup/files/init.d/010-languages.sh +++ b/charts/dataverseup/files/init.d/010-languages.sh @@ -1 +1 @@ -../../../../init.d/010-languages.sh \ No newline at end of file +../../../../scripts/init.d/010-languages.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-mailrelay-set.sh b/charts/dataverseup/files/init.d/010-mailrelay-set.sh index da144d5..7830b52 120000 --- a/charts/dataverseup/files/init.d/010-mailrelay-set.sh +++ b/charts/dataverseup/files/init.d/010-mailrelay-set.sh @@ -1 +1 @@ -../../../../init.d/010-mailrelay-set.sh \ No newline at end of file +../../../../scripts/init.d/010-mailrelay-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/011-local-storage.sh b/charts/dataverseup/files/init.d/011-local-storage.sh index e16e2d0..59c1676 120000 --- a/charts/dataverseup/files/init.d/011-local-storage.sh +++ b/charts/dataverseup/files/init.d/011-local-storage.sh @@ -1 +1 @@ -../../../../init.d/011-local-storage.sh \ No newline at end of file +../../../../scripts/init.d/011-local-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/012-minio-bucket1.sh b/charts/dataverseup/files/init.d/012-minio-bucket1.sh index 9c70fdf..e4f3db6 120000 --- a/charts/dataverseup/files/init.d/012-minio-bucket1.sh +++ b/charts/dataverseup/files/init.d/012-minio-bucket1.sh @@ -1 +1 @@ -../../../../init.d/012-minio-bucket1.sh \ No newline at end of file +../../../../scripts/init.d/012-minio-bucket1.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/013-minio-bucket2.sh b/charts/dataverseup/files/init.d/013-minio-bucket2.sh index 775442a..fe3c9d8 120000 --- a/charts/dataverseup/files/init.d/013-minio-bucket2.sh +++ b/charts/dataverseup/files/init.d/013-minio-bucket2.sh @@ -1 +1 @@ -../../../../init.d/013-minio-bucket2.sh \ No newline at end of file +../../../../scripts/init.d/013-minio-bucket2.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/02-controlled-voc.sh b/charts/dataverseup/files/init.d/02-controlled-voc.sh index 816a86c..aaa29e8 120000 --- a/charts/dataverseup/files/init.d/02-controlled-voc.sh +++ b/charts/dataverseup/files/init.d/02-controlled-voc.sh @@ -1 +1 @@ -../../../../init.d/02-controlled-voc.sh \ No newline at end of file +../../../../scripts/init.d/02-controlled-voc.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/03-doi-set.sh b/charts/dataverseup/files/init.d/03-doi-set.sh index 0d02c60..59f2e33 120000 --- a/charts/dataverseup/files/init.d/03-doi-set.sh +++ b/charts/dataverseup/files/init.d/03-doi-set.sh @@ -1 +1 @@ -../../../../init.d/03-doi-set.sh \ No newline at end of file +../../../../scripts/init.d/03-doi-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/04-setdomain.sh b/charts/dataverseup/files/init.d/04-setdomain.sh index 791a335..da0ff7a 120000 --- a/charts/dataverseup/files/init.d/04-setdomain.sh +++ b/charts/dataverseup/files/init.d/04-setdomain.sh @@ -1 +1 @@ -../../../../init.d/04-setdomain.sh \ No newline at end of file +../../../../scripts/init.d/04-setdomain.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/05-reindex.sh b/charts/dataverseup/files/init.d/05-reindex.sh index c256126..0e0087a 120000 --- a/charts/dataverseup/files/init.d/05-reindex.sh +++ b/charts/dataverseup/files/init.d/05-reindex.sh @@ -1 +1 @@ -../../../../init.d/05-reindex.sh \ No newline at end of file +../../../../scripts/init.d/05-reindex.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/07-previewers.sh b/charts/dataverseup/files/init.d/07-previewers.sh index 804ac47..d1867a7 120000 --- a/charts/dataverseup/files/init.d/07-previewers.sh +++ b/charts/dataverseup/files/init.d/07-previewers.sh @@ -1 +1 @@ -../../../../init.d/07-previewers.sh \ No newline at end of file +../../../../scripts/init.d/07-previewers.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/08-federated-login.sh b/charts/dataverseup/files/init.d/08-federated-login.sh index 43463b7..f9e9344 120000 --- a/charts/dataverseup/files/init.d/08-federated-login.sh +++ b/charts/dataverseup/files/init.d/08-federated-login.sh @@ -1 +1 @@ -../../../../init.d/08-federated-login.sh \ No newline at end of file +../../../../scripts/init.d/08-federated-login.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1001-webhooks.sh b/charts/dataverseup/files/init.d/1001-webhooks.sh index 66eaefe..495712a 120000 --- a/charts/dataverseup/files/init.d/1001-webhooks.sh +++ b/charts/dataverseup/files/init.d/1001-webhooks.sh @@ -1 +1 @@ -../../../../init.d/1001-webhooks.sh \ No newline at end of file +../../../../scripts/init.d/1001-webhooks.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1002-custom-metadata.sh b/charts/dataverseup/files/init.d/1002-custom-metadata.sh index 05b8703..479c9b6 120000 --- a/charts/dataverseup/files/init.d/1002-custom-metadata.sh +++ b/charts/dataverseup/files/init.d/1002-custom-metadata.sh @@ -1 +1 @@ -../../../../init.d/1002-custom-metadata.sh \ No newline at end of file +../../../../scripts/init.d/1002-custom-metadata.sh \ No newline at end of file diff --git a/charts/dataverseup/templates/_helpers.tpl b/charts/dataverseup/templates/_helpers.tpl index 61edcda..7b74311 100644 --- a/charts/dataverseup/templates/_helpers.tpl +++ b/charts/dataverseup/templates/_helpers.tpl @@ -64,6 +64,83 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} +{{/* +ConfigMap for compose-style bootstrap: bootstrap-chain.sh, apply-branding.sh, seed-content.sh, branding.env, seed fixtures. +*/}} +{{- define "dataverseup.bootstrapChainConfigMapName" -}} +{{- printf "%s-bootstrap-chain" (include "dataverseup.fullname" .) }} +{{- end }} + +{{/* +Volume mounts for compose-mode bootstrap Jobs (configbaker runs bootstrap-chain.sh from the chain ConfigMap). +*/}} +{{- define "dataverseup.bootstrapComposeVolumeMounts" -}} +- name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true +- name: bootstrap-work + mountPath: /work +- name: branding-env + mountPath: /config + readOnly: true +- name: seed-flat + mountPath: /seed-flat + readOnly: true +{{- end }} + +{{/* +Volumes for compose-mode bootstrap (chain ConfigMap + emptyDir workdir). +*/}} +{{- define "dataverseup.bootstrapComposeVolumes" -}} +- name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh +- name: branding-env + configMap: + name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} + items: + - key: branding.env + path: branding.env +- name: seed-flat + configMap: + name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv +- name: bootstrap-work + emptyDir: {} +{{- end }} + +{{/* +Minimal env for default bootstrap Job (DATAVERSE_URL + TIMEOUT). Dict keys: dvUrl, timeout. +*/}} +{{- define "dataverseup.bootstrapJobMinimalEnv" -}} +- name: DATAVERSE_URL + value: {{ index . "dvUrl" | quote }} +- name: TIMEOUT + value: {{ index . "timeout" | quote }} +{{- end -}} + {{/* Service / CLI label query for the main Dataverse pods only. Pods also set component=primary; Deployment matchLabels stay name+instance only so upgrades do not hit immutable selector changes. diff --git a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml index 4874f01..31a20ee 100644 --- a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml +++ b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} labels: {{- include "dataverseup.labels" . | nindent 4 }} app.kubernetes.io/component: bootstrap-chain diff --git a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml index f2e69dd..0cc7b57 100644 --- a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml +++ b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml @@ -59,57 +59,11 @@ spec: name: {{ $pu.existingSecret | quote }} key: {{ $pu.secretKey | quote }} volumeMounts: - - name: bootstrap-scripts - mountPath: /bootstrap-chain - readOnly: true - - name: bootstrap-work - mountPath: /work - - name: branding-env - mountPath: /config - readOnly: true - - name: seed-flat - mountPath: /seed-flat - readOnly: true +{{ include "dataverseup.bootstrapComposeVolumeMounts" . | nindent 12 }} {{- with .Values.bootstrapJob.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} volumes: - - name: bootstrap-scripts - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - defaultMode: 0555 - items: - - key: bootstrap-chain.sh - path: bootstrap-chain.sh - - key: apply-branding.sh - path: apply-branding.sh - - key: seed-content.sh - path: seed-content.sh - - name: branding-env - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - items: - - key: branding.env - path: branding.env - - name: seed-flat - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - items: - - key: demo-collection.json - path: demo-collection.json - - key: dataset-images.json - path: dataset-images.json - - key: dataset-tabular.json - path: dataset-tabular.json - - key: files_1x1.png - path: files_1x1.png - - key: files_badge.svg - path: files_badge.svg - - key: files_readme.txt - path: files_readme.txt - - key: files_sample.csv - path: files_sample.csv - - name: bootstrap-work - emptyDir: {} +{{ include "dataverseup.bootstrapComposeVolumes" . | nindent 8 }} {{- end }} diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml index e3f989a..be9ed95 100644 --- a/charts/dataverseup/templates/bootstrap-job.yaml +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -62,24 +62,11 @@ spec: optional: true {{- end }} volumeMounts: - - name: bootstrap-scripts - mountPath: /bootstrap-chain - readOnly: true - - name: bootstrap-work - mountPath: /work - - name: branding-env - mountPath: /config - readOnly: true - - name: seed-flat - mountPath: /seed-flat - readOnly: true +{{ include "dataverseup.bootstrapComposeVolumeMounts" . | nindent 12 }} {{- else }} command: {{- toYaml .Values.bootstrapJob.command | nindent 12 }} env: - - name: DATAVERSE_URL - value: {{ $dvUrl | quote }} - - name: TIMEOUT - value: {{ .Values.bootstrapJob.timeout | quote }} +{{ include "dataverseup.bootstrapJobMinimalEnv" (dict "dvUrl" $dvUrl "timeout" .Values.bootstrapJob.timeout) | indent 12 }} {{- end }} {{- with .Values.bootstrapJob.resources }} resources: @@ -87,42 +74,6 @@ spec: {{- end }} {{- if eq .Values.bootstrapJob.mode "compose" }} volumes: - - name: bootstrap-scripts - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - defaultMode: 0555 - items: - - key: bootstrap-chain.sh - path: bootstrap-chain.sh - - key: apply-branding.sh - path: apply-branding.sh - - key: seed-content.sh - path: seed-content.sh - - name: branding-env - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - items: - - key: branding.env - path: branding.env - - name: seed-flat - configMap: - name: {{ include "dataverseup.fullname" . }}-bootstrap-chain - items: - - key: demo-collection.json - path: demo-collection.json - - key: dataset-images.json - path: dataset-images.json - - key: dataset-tabular.json - path: dataset-tabular.json - - key: files_1x1.png - path: files_1x1.png - - key: files_badge.svg - path: files_badge.svg - - key: files_readme.txt - path: files_readme.txt - - key: files_sample.csv - path: files_sample.csv - - name: bootstrap-work - emptyDir: {} +{{ include "dataverseup.bootstrapComposeVolumes" . | nindent 8 }} {{- end }} {{- end }} diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml index 0013ca7..a51f5ec 100644 --- a/charts/dataverseup/values.yaml +++ b/charts/dataverseup/values.yaml @@ -153,7 +153,7 @@ brandingNavbarLogos: docrootUid: 1000 docrootGid: 1000 -# Ship the repo **init.d/*.sh** bundle into the Payara ConfigMap (compose parity: ./init.d → /opt/payara/init.d). +# Ship the repo **scripts/init.d/*.sh** bundle into the Payara ConfigMap (compose parity: ./scripts/init.d → /opt/payara/init.d). # When true, all scripts under charts/dataverseup/files/init.d/*.sh are included; optional configMap.data overlays. # Some scripts assume compose paths (MinIO, /opt/payara/triggers, etc.) — review before enabling in production. initdFromChart: diff --git a/docker-compose.yml b/docker-compose.yml index db08f14..edabcbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - "POSTGRES_PORT" volumes: - postgres_data:/var/lib/postgresql/data - - ./triggers:/triggers + - ./scripts/triggers:/triggers solr: <<: *env-from-file @@ -184,8 +184,8 @@ services: # /dv/docroot/logos//... (theme uploads) stays on the container filesystem. - ./branding/docroot/logos/navbar:/dv/docroot/logos/navbar:ro - ./branding/docroot/branding:/dv/docroot/branding:ro - - ./init.d:/opt/payara/init.d - - ./triggers:/opt/payara/triggers + - ./scripts/init.d:/opt/payara/init.d + - ./scripts/triggers:/opt/payara/triggers - ./config/schema.xml:/opt/payara/dvinstall/schema.xml labels: - "traefik.enable=true" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 625fdb7..51f069e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -165,9 +165,9 @@ If you terminate TLS or expose the app on a **non-default host port**, keep **`D ### Payara init scripts (DRY with Compose) -The chart embeds the S3 and mail relay scripts from **`init.d/`** at the repo root via symlinks under `charts/dataverseup/files/`. Edit **`init.d/006-s3-aws-storage.sh`** or **`init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. +The chart embeds the S3 and mail relay scripts from **`scripts/init.d/`** via symlinks under `charts/dataverseup/files/init.d/`. Edit **`scripts/init.d/006-s3-aws-storage.sh`** or **`scripts/init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. -Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting `./init.d`). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and triggers-specific scripts before enabling in a cluster that does not mount those paths. +Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting **`./scripts/init.d`**). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and webhook/trigger scripts (repo **`scripts/triggers/`**, Compose → `/opt/payara/triggers`) before enabling in a cluster that does not mount those paths. ### S3 file storage @@ -214,7 +214,7 @@ Set **`initdFromChart.enabled: true`** in values to include **all** `files/init. The Native API returns **HTTP 400** with that message when the request reached Dataverse but **writing to the configured store failed**. This is not a bug in the seed script’s `jsonData` shape. -With **`dataverse.files.storage-driver-id=S3`** (see `init.d/006-s3-aws-storage.sh`): +With **`dataverse.files.storage-driver-id=S3`** (see `scripts/init.d/006-s3-aws-storage.sh`): 1. **IAM** — The principal in your `aws-s3-credentials` Secret needs at least **`s3:PutObject`**, **`s3:GetObject`**, **`s3:DeleteObject`**, and **`s3:ListBucket`** on the target bucket (and prefixes Dataverse uses). Missing `PutObject` often surfaces exactly as this generic message; the real error is in server logs. 2. **Bucket and region** — `awsS3.bucketName` and `awsS3.region` must match the bucket. For **Amazon S3**, keep **`awsS3.endpointUrl` empty**; do not point it at `https://s3..amazonaws.com` unless you are on a non-AWS S3-compatible store. diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 3dc30b3..01f98ca 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -172,14 +172,14 @@ solrInit: adminPassword: "" resources: {} -# Compose mounts the full repo **init.d/** at `/opt/payara/init.d`. Enable this to ship the same `*.sh` bundle via +# Compose mounts **scripts/init.d/** at `/opt/payara/init.d`. Enable this to ship the same `*.sh` bundle via # the chart ConfigMap (see **initdFromChart** in **charts/dataverseup/values.yaml**). Scripts run in lexical order; -# **012/013-minio** expect compose MinIO (no-op or skip if you use S3 only). **1001-webhooks** needs **triggers/** +# **012/013-minio** expect compose MinIO (no-op or skip if you use S3 only). **1001-webhooks** needs **scripts/triggers/** mounted at `/opt/payara/triggers` (Compose default). # on disk unless you mount it separately. initdFromChart: enabled: true -# Mount init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. +# Mount scripts/init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. mail: enabled: true @@ -214,7 +214,7 @@ extraEnvVars: value: "8983" - name: dataverse_solr_core value: dataverse - # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style init.d + # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style scripts/init.d # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. - name: DATAVERSE_URL value: "http://demo-dataverseup.demo-dataverseup.svc.cluster.local:80" @@ -244,7 +244,7 @@ extraEnvVars: value: FK2/ - name: INIT_SCRIPTS_FOLDER value: /opt/payara/init.d - # Payara JavaMail + :SystemEmail (init.d/010-mailrelay-set.sh). + # Payara JavaMail + :SystemEmail (scripts/init.d/010-mailrelay-set.sh). # Literal SMTP_* (override via GitHub Environment if you reintroduce envsubst for these keys). - name: SMTP_ADDRESS value: "smtp.sendgrid.net" diff --git a/init.d/006-s3-aws-storage.sh b/scripts/init.d/006-s3-aws-storage.sh similarity index 100% rename from init.d/006-s3-aws-storage.sh rename to scripts/init.d/006-s3-aws-storage.sh diff --git a/init.d/01-persistent-id.sh b/scripts/init.d/01-persistent-id.sh similarity index 100% rename from init.d/01-persistent-id.sh rename to scripts/init.d/01-persistent-id.sh diff --git a/init.d/010-languages.sh b/scripts/init.d/010-languages.sh similarity index 100% rename from init.d/010-languages.sh rename to scripts/init.d/010-languages.sh diff --git a/init.d/010-mailrelay-set.sh b/scripts/init.d/010-mailrelay-set.sh similarity index 93% rename from init.d/010-mailrelay-set.sh rename to scripts/init.d/010-mailrelay-set.sh index 3cb5f5f..5fe74c7 100644 --- a/init.d/010-mailrelay-set.sh +++ b/scripts/init.d/010-mailrelay-set.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Setup mail relay (Docker Compose init.d + Helm ConfigMap when values.mail.enabled). +# Setup mail relay (Compose mount scripts/init.d + Helm ConfigMap when values.mail.enabled). # https://guides.dataverse.org/en/latest/developers/troubleshooting.html # # smtp_enabled: false|0|no skips. smtp_type=plain can imply SMTP AUTH when smtp_auth is unset. diff --git a/init.d/011-local-storage.sh b/scripts/init.d/011-local-storage.sh similarity index 100% rename from init.d/011-local-storage.sh rename to scripts/init.d/011-local-storage.sh diff --git a/init.d/012-minio-bucket1.sh b/scripts/init.d/012-minio-bucket1.sh similarity index 100% rename from init.d/012-minio-bucket1.sh rename to scripts/init.d/012-minio-bucket1.sh diff --git a/init.d/013-minio-bucket2.sh b/scripts/init.d/013-minio-bucket2.sh similarity index 100% rename from init.d/013-minio-bucket2.sh rename to scripts/init.d/013-minio-bucket2.sh diff --git a/init.d/02-controlled-voc.sh b/scripts/init.d/02-controlled-voc.sh similarity index 100% rename from init.d/02-controlled-voc.sh rename to scripts/init.d/02-controlled-voc.sh diff --git a/init.d/03-doi-set.sh b/scripts/init.d/03-doi-set.sh similarity index 100% rename from init.d/03-doi-set.sh rename to scripts/init.d/03-doi-set.sh diff --git a/init.d/04-setdomain.sh b/scripts/init.d/04-setdomain.sh similarity index 100% rename from init.d/04-setdomain.sh rename to scripts/init.d/04-setdomain.sh diff --git a/init.d/05-reindex.sh b/scripts/init.d/05-reindex.sh similarity index 100% rename from init.d/05-reindex.sh rename to scripts/init.d/05-reindex.sh diff --git a/init.d/07-previewers.sh b/scripts/init.d/07-previewers.sh similarity index 100% rename from init.d/07-previewers.sh rename to scripts/init.d/07-previewers.sh diff --git a/init.d/08-federated-login.sh b/scripts/init.d/08-federated-login.sh similarity index 100% rename from init.d/08-federated-login.sh rename to scripts/init.d/08-federated-login.sh diff --git a/init.d/1001-webhooks.sh b/scripts/init.d/1001-webhooks.sh similarity index 100% rename from init.d/1001-webhooks.sh rename to scripts/init.d/1001-webhooks.sh diff --git a/init.d/1002-custom-metadata.sh b/scripts/init.d/1002-custom-metadata.sh similarity index 93% rename from init.d/1002-custom-metadata.sh rename to scripts/init.d/1002-custom-metadata.sh index 6f4e22b..46c976b 100755 --- a/init.d/1002-custom-metadata.sh +++ b/scripts/init.d/1002-custom-metadata.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -# Solr field-update scripts are vendored under init.d/vendor-solr/ (from IQSS Dataverse v5.x release assets; stable helpers for schema merge + Solr MDB update). +# Solr field-update scripts are vendored under vendor-solr/ next to this script (repo: scripts/init.d/vendor-solr/; container: $HOME_DIR/init.d/vendor-solr/). # Avoids runtime version skew against the running Dataverse image. if [ "${CLARIN:-}" ]; then diff --git a/init.d/vendor-solr/update-fields.sh b/scripts/init.d/vendor-solr/update-fields.sh similarity index 100% rename from init.d/vendor-solr/update-fields.sh rename to scripts/init.d/vendor-solr/update-fields.sh diff --git a/init.d/vendor-solr/updateSchemaMDB.sh b/scripts/init.d/vendor-solr/updateSchemaMDB.sh similarity index 100% rename from init.d/vendor-solr/updateSchemaMDB.sh rename to scripts/init.d/vendor-solr/updateSchemaMDB.sh diff --git a/config/update-fields.sh b/scripts/solr/update-fields.sh similarity index 100% rename from config/update-fields.sh rename to scripts/solr/update-fields.sh diff --git a/triggers/affiliations.sql b/scripts/triggers/affiliations.sql similarity index 100% rename from triggers/affiliations.sql rename to scripts/triggers/affiliations.sql diff --git a/triggers/external-service.sql b/scripts/triggers/external-service.sql similarity index 100% rename from triggers/external-service.sql rename to scripts/triggers/external-service.sql diff --git a/triggers/external-services.py b/scripts/triggers/external-services.py similarity index 100% rename from triggers/external-services.py rename to scripts/triggers/external-services.py diff --git a/triggers/lang-properties-convert.py b/scripts/triggers/lang-properties-convert.py similarity index 100% rename from triggers/lang-properties-convert.py rename to scripts/triggers/lang-properties-convert.py From 4158b46767fb753c98af6675bb578a3970ccca70 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Fri, 3 Apr 2026 23:38:55 -0700 Subject: [PATCH 29/31] Smtp mail setup --- .env.example | 13 ++++ .github/workflows/deploy.yaml | 4 +- README.md | 5 ++ charts/dataverseup/templates/_helpers.tpl | 77 ------------------- .../templates/bootstrap-chain-configmap.yaml | 2 +- .../bootstrap-compose-postupgrade-job.yaml | 50 +++++++++++- .../dataverseup/templates/bootstrap-job.yaml | 55 ++++++++++++- docs/DEPLOYMENT.md | 23 +++++- ops/demo-deploy.tmpl.yaml | 10 +-- 9 files changed, 146 insertions(+), 93 deletions(-) diff --git a/.env.example b/.env.example index 8b29357..eb8c5b2 100644 --- a/.env.example +++ b/.env.example @@ -39,6 +39,19 @@ MINIO_ROOT_PASSWORD=love1234 # Optional; unused for local dev when TLS is provided by `sc proxy` (Stack Car). useremail=you@example.org +# --- Optional: SMTP (scripts/init.d/010-mailrelay-set.sh; requires system_email; smtp_enabled not false/0/no) --- +# smtp_enabled=true +# smtp_type=plain +# smtp_auth=true +# smtp_starttls=true +# smtp_port=587 +# socket_port=587 +# mailhost=smtp.example.com +# mailuser=apikey-or-username +# smtp_password=secret +# system_email=support@example.com +# no_reply_email=noreply@example.com + WEBHOOK=/opt/payara/triggers/external-services.py # Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / scripts/init.d) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 8d12453..3a4aeb6 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -46,7 +46,6 @@ jobs: SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} - MAIL_SMTP_PASSWORD: ${{ secrets.MAIL_SMTP_PASSWORD }} SMTP_ADDRESS: ${{ vars.SMTP_ADDRESS }} SMTP_USER_NAME: ${{ vars.SMTP_USER_NAME }} SMTP_PORT: ${{ vars.SMTP_PORT }} @@ -96,12 +95,11 @@ jobs: echo "$KUBECONFIG_FILE" | base64 -d >"$KUBECONFIG" export SMTP_PORT="${SMTP_PORT:-25}" export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" - export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" fi # Only secrets + rollout id — hosts, Solr DNS, ingress, and bucket are literals in the *.tmpl.yaml file. - ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' + ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $NO_REPLY_EMAIL $SMTP_PASSWORD $SMTP_AUTH' envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT" - name: Solr conf ConfigMap (pre-Helm) diff --git a/README.md b/README.md index f23a1ff..6b5e4d6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and a **Helm chart** for Kubernetes without forking core. +| | | +|--|--| +| **Repository** | [github.com/notch8/dataverseup](https://github.com/notch8/dataverseup) | +| **Live demo** | [demo-dataverseup.notch8.cloud](https://demo-dataverseup.notch8.cloud/) — deployed from this stack (Helm/Kubernetes); seeded demo data for smoke tests | + ## Quick start (local / lab) 1. **Prerequisites:** Docker + Docker Compose v2 (`docker compose`), ~4 GB+ RAM (Payara + Solr + Postgres), and **Ruby + RubyGems** for [Stack Car](https://rubygems.org/gems/stack_car) (edge TLS — same **`*.localhost.direct`** pattern as Hyku). On Apple Silicon, images use `linux/amd64` (emulation). diff --git a/charts/dataverseup/templates/_helpers.tpl b/charts/dataverseup/templates/_helpers.tpl index 7b74311..61edcda 100644 --- a/charts/dataverseup/templates/_helpers.tpl +++ b/charts/dataverseup/templates/_helpers.tpl @@ -64,83 +64,6 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} -{{/* -ConfigMap for compose-style bootstrap: bootstrap-chain.sh, apply-branding.sh, seed-content.sh, branding.env, seed fixtures. -*/}} -{{- define "dataverseup.bootstrapChainConfigMapName" -}} -{{- printf "%s-bootstrap-chain" (include "dataverseup.fullname" .) }} -{{- end }} - -{{/* -Volume mounts for compose-mode bootstrap Jobs (configbaker runs bootstrap-chain.sh from the chain ConfigMap). -*/}} -{{- define "dataverseup.bootstrapComposeVolumeMounts" -}} -- name: bootstrap-scripts - mountPath: /bootstrap-chain - readOnly: true -- name: bootstrap-work - mountPath: /work -- name: branding-env - mountPath: /config - readOnly: true -- name: seed-flat - mountPath: /seed-flat - readOnly: true -{{- end }} - -{{/* -Volumes for compose-mode bootstrap (chain ConfigMap + emptyDir workdir). -*/}} -{{- define "dataverseup.bootstrapComposeVolumes" -}} -- name: bootstrap-scripts - configMap: - name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} - defaultMode: 0555 - items: - - key: bootstrap-chain.sh - path: bootstrap-chain.sh - - key: apply-branding.sh - path: apply-branding.sh - - key: seed-content.sh - path: seed-content.sh -- name: branding-env - configMap: - name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} - items: - - key: branding.env - path: branding.env -- name: seed-flat - configMap: - name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} - items: - - key: demo-collection.json - path: demo-collection.json - - key: dataset-images.json - path: dataset-images.json - - key: dataset-tabular.json - path: dataset-tabular.json - - key: files_1x1.png - path: files_1x1.png - - key: files_badge.svg - path: files_badge.svg - - key: files_readme.txt - path: files_readme.txt - - key: files_sample.csv - path: files_sample.csv -- name: bootstrap-work - emptyDir: {} -{{- end }} - -{{/* -Minimal env for default bootstrap Job (DATAVERSE_URL + TIMEOUT). Dict keys: dvUrl, timeout. -*/}} -{{- define "dataverseup.bootstrapJobMinimalEnv" -}} -- name: DATAVERSE_URL - value: {{ index . "dvUrl" | quote }} -- name: TIMEOUT - value: {{ index . "timeout" | quote }} -{{- end -}} - {{/* Service / CLI label query for the main Dataverse pods only. Pods also set component=primary; Deployment matchLabels stay name+instance only so upgrades do not hit immutable selector changes. diff --git a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml index 31a20ee..4874f01 100644 --- a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml +++ b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "dataverseup.bootstrapChainConfigMapName" . }} + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain labels: {{- include "dataverseup.labels" . | nindent 4 }} app.kubernetes.io/component: bootstrap-chain diff --git a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml index 0cc7b57..f2e69dd 100644 --- a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml +++ b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml @@ -59,11 +59,57 @@ spec: name: {{ $pu.existingSecret | quote }} key: {{ $pu.secretKey | quote }} volumeMounts: -{{ include "dataverseup.bootstrapComposeVolumeMounts" . | nindent 12 }} + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true {{- with .Values.bootstrapJob.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} volumes: -{{ include "dataverseup.bootstrapComposeVolumes" . | nindent 8 }} + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} {{- end }} diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml index be9ed95..e3f989a 100644 --- a/charts/dataverseup/templates/bootstrap-job.yaml +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -62,11 +62,24 @@ spec: optional: true {{- end }} volumeMounts: -{{ include "dataverseup.bootstrapComposeVolumeMounts" . | nindent 12 }} + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true {{- else }} command: {{- toYaml .Values.bootstrapJob.command | nindent 12 }} env: -{{ include "dataverseup.bootstrapJobMinimalEnv" (dict "dvUrl" $dvUrl "timeout" .Values.bootstrapJob.timeout) | indent 12 }} + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} {{- end }} {{- with .Values.bootstrapJob.resources }} resources: @@ -74,6 +87,42 @@ spec: {{- end }} {{- if eq .Values.bootstrapJob.mode "compose" }} volumes: -{{ include "dataverseup.bootstrapComposeVolumes" . | nindent 8 }} + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} {{- end }} {{- end }} diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 51f069e..9e1dbc8 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -2,6 +2,8 @@ Rough notes for standing up Dataverse for Notch8: **Docker Compose** (lab), **Kubernetes / Helm**, optional **GitHub Actions** deploys, and a shared **learnings** table. Extend into a full runbook as you validate each environment. +**Reference:** Source — [github.com/notch8/dataverseup](https://github.com/notch8/dataverseup). An **active** deployment from this approach is at [demo-dataverseup.notch8.cloud](https://demo-dataverseup.notch8.cloud/) (seeded demo content for smoke tests). + ## Ticket context (internal) - **Target:** Dataverse **v6.10** on **AWS** by **April 7, 2026** — functional demo, not necessarily production-hardened. @@ -139,9 +141,9 @@ Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core afte ### GitHub Actions — Deploy workflow -The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for secrets** (`DB_PASSWORD`, `SYSTEM_EMAIL`, `SMTP_PASSWORD`, `SMTP_AUTH`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). +The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for selected secrets** (`DB_PASSWORD`, `SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`, `SMTP_AUTH`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). -**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail secrets (`SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`, `MAIL_SMTP_PASSWORD`). +**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail: `SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`. **Repository or Environment variables (optional):** @@ -152,11 +154,28 @@ The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job us | `HELM_APP_NAME` | `app.kubernetes.io/name` for `kubectl rollout status` | `github.event.repository.name` | | `DEPLOY_ROLLOUT_TIMEOUT` | Rollout wait | `10m` | | `DEPLOY_BOOTSTRAP_JOB_TIMEOUT` | Bootstrap Job wait | `25m` | +| `SMTP_ADDRESS` | Host for `mailhost` in values (e.g. SendGrid) | (use literals in `ops/*-deploy.tmpl.yaml` if not set) | +| `SMTP_USER_NAME` | `mailuser` (e.g. `apikey` for SendGrid) | | +| `SMTP_PORT` / `SOCKET_PORT` | Ports for JavaMail | workflow defaults `SMTP_PORT` to `25` if unset | +| `SMTP_AUTH` | Must be `true` for authenticated SMTP (injected by `envsubst` into `smtp_auth`) | | +| `SMTP_STARTTLS`, `SMTP_TYPE`, `SMTP_ENABLED`, `SMTP_DOMAIN` | Passed through to job env; if `NO_REPLY_EMAIL` is empty but `SMTP_DOMAIN` is set, the workflow sets `NO_REPLY_EMAIL=noreply@$SMTP_DOMAIN` before `envsubst` | | Default Helm **release** and **namespace** are **`-`** (e.g. `demo-dataverseup`). Override with workflow inputs `k8s_release_name` / `k8s_namespace` when needed. **Migrating or renaming a release:** Update the literals in **`ops/-deploy.tmpl.yaml`** (ingress hosts, `dataverse_*` / `DATAVERSE_*` / `hostname`, `solrHttpBase`, `SOLR_*`, `DATAVERSE_URL`, `awsS3.bucketName`, DB names, etc.) so they match the new Helm release and namespace; then align Postgres, S3, TLS, and running workloads. +#### SMTP (JavaMail) — enable, secrets, and how to test + +1. **Chart:** set **`mail.enabled: true`** so **`010-mailrelay-set.sh`** is in the ConfigMap. Pod env must include the names the script reads: **`system_email`**, **`mailhost`**, **`mailuser`**, **`no_reply_email`**, **`smtp_password`**, **`smtp_port`**, **`socket_port`**, **`smtp_auth`**, **`smtp_starttls`**, **`smtp_type`**, **`smtp_enabled`** (see **`ops/demo-deploy.tmpl.yaml`** and **`scripts/init.d/010-mailrelay-set.sh`**). **`smtp_enabled`** must not be `false` / `0` / `no`, and **`system_email`** must be non-empty or the script no-ops. + +2. **GitHub Environment (demo):** Add **Secrets** — `SMTP_PASSWORD` (SendGrid API key or provider password), `SYSTEM_EMAIL` (installation support address; `:SystemEmail`), `NO_REPLY_EMAIL` (JavaMail **From** / `no_reply_email`). Optionally set **Variables** (`SMTP_ADDRESS`, `SMTP_USER_NAME`, `SMTP_PORT`, `SMTP_AUTH=true`, etc.) if you do not want them as literals in the template. Redeploy so a **new pod** runs init (the script uses **`asadmin create-javamail-resource`** on boot). + +3. **Verify config:** After the pod is ready, check the setting: `curl -sS "https:///api/admin/settings/:SystemEmail"` (use a superuser API token if the endpoint requires auth on your version). In the UI, use **Contact** / support mail, **Forgot password**, or user signup (if enabled) and confirm delivery (and provider dashboard / spam folder). + +4. **Logs:** `kubectl logs -n deploy/-dataverseup` (or your deployment name) and look for **`010-mailrelay`** / **`asadmin`** errors during startup. + +5. **Docker Compose (local):** Set the same variable names in **`.env`** (see **`.env.example`** SMTP block). They are passed through on the **dataverse** service. Restart **dataverse** and tail **`docker compose logs -f dataverse`** during Payara init. + ### Ingress and TLS Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 01f98ca..10843ba 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -5,10 +5,10 @@ # They must stay consistent with your Helm **release name** and **namespace** (e.g. both `demo-dataverseup`). # # **envsubst** (CI: see `.github/workflows/deploy.yaml`) only injects secrets and the rollout nonce — not URLs: -# $DB_PASSWORD, $SYSTEM_EMAIL, $SMTP_PASSWORD, $SMTP_AUTH, ${GITHUB_RUN_ID} +# $DB_PASSWORD, $SYSTEM_EMAIL, $NO_REPLY_EMAIL, $SMTP_PASSWORD, $SMTP_AUTH, ${GITHUB_RUN_ID} # Local render: -# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SYSTEM_EMAIL=... SMTP_PASSWORD=... SMTP_AUTH= -# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml +# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SYSTEM_EMAIL=... NO_REPLY_EMAIL=... SMTP_PASSWORD=... SMTP_AUTH= +# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $NO_REPLY_EMAIL $SMTP_PASSWORD $SMTP_AUTH' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml # # Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), # same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/DEPLOYMENT.md**). @@ -264,7 +264,7 @@ extraEnvVars: # no_reply_email → Payara JavaMail --fromaddress (outbound From). No separate Reply-To in the session; # mail clients reply to From when Reply-To is absent, so use support@ for both behaviors. - name: system_email - value: "${SYSTEM_EMAIL}" + value: "support@notch8.com" - name: mailhost value: "smtp.sendgrid.net" - name: mailuser @@ -278,7 +278,7 @@ extraEnvVars: - name: socket_port value: "587" - name: smtp_auth - value: "${SMTP_AUTH}" + value: "true" - name: smtp_starttls value: "true" - name: smtp_type From 9483b28933c98a173b2473d8765da1a5e05b0d42 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 4 Apr 2026 00:03:37 -0700 Subject: [PATCH 30/31] align smtp --- .github/workflows/deploy.yaml | 22 +++------------------- docs/DEPLOYMENT.md | 25 +++++++++++++++++-------- ops/demo-deploy.tmpl.yaml | 11 +++++------ 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 3a4aeb6..fae6bb9 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -42,19 +42,8 @@ jobs: environment: ${{ inputs.environment }} env: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - # Optional mail — secrets + Environment variables (see ops/demo-deploy.tmpl.yaml header). - SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} - NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} + # Mail: SendGrid API key only — host/ports/From/support address are literals in ops/-deploy.tmpl.yaml. SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} - SMTP_ADDRESS: ${{ vars.SMTP_ADDRESS }} - SMTP_USER_NAME: ${{ vars.SMTP_USER_NAME }} - SMTP_PORT: ${{ vars.SMTP_PORT }} - SOCKET_PORT: ${{ vars.SOCKET_PORT }} - SMTP_AUTH: ${{ vars.SMTP_AUTH }} - SMTP_STARTTLS: ${{ vars.SMTP_STARTTLS }} - SMTP_TYPE: ${{ vars.SMTP_TYPE }} - SMTP_ENABLED: ${{ vars.SMTP_ENABLED }} - SMTP_DOMAIN: ${{ vars.SMTP_DOMAIN }} DEPLOY_ENVIRONMENT: ${{ inputs.environment }} HELM_RELEASE_NAME: ${{ inputs.k8s_release_name || format('{0}-{1}', inputs.environment, github.event.repository.name) }} HELM_NAMESPACE: ${{ inputs.k8s_namespace || format('{0}-{1}', inputs.environment, github.event.repository.name) }} @@ -93,13 +82,8 @@ jobs: TMPL="ops/${DEPLOY_ENVIRONMENT}-deploy.tmpl.yaml" OUT="ops/${DEPLOY_ENVIRONMENT}-deploy.yaml" echo "$KUBECONFIG_FILE" | base64 -d >"$KUBECONFIG" - export SMTP_PORT="${SMTP_PORT:-25}" - export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" - if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then - export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" - fi - # Only secrets + rollout id — hosts, Solr DNS, ingress, and bucket are literals in the *.tmpl.yaml file. - ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $NO_REPLY_EMAIL $SMTP_PASSWORD $SMTP_AUTH' + # envsubst: only secrets + rollout id. Everything else (SMTP host, ports, From, URLs, Solr, …) is literals in the *.tmpl.yaml file. + ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SMTP_PASSWORD' envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT" - name: Solr conf ConfigMap (pre-Helm) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9e1dbc8..3321e9a 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -141,9 +141,9 @@ Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core afte ### GitHub Actions — Deploy workflow -The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for selected secrets** (`DB_PASSWORD`, `SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`, `SMTP_AUTH`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). +The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for selected secrets** (`DB_PASSWORD`, `SMTP_PASSWORD`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, `:SystemEmail` / JavaMail From addresses (demo: both `support@notch8.com`), S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). -**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail: `SYSTEM_EMAIL`, `NO_REPLY_EMAIL`, `SMTP_PASSWORD`. +**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail: **`SMTP_PASSWORD`** (SendGrid API key for demo). Demo values fix **`system_email`** and **`no_reply_email`** to **`support@notch8.com`** in **`ops/demo-deploy.tmpl.yaml`** (no `SYSTEM_EMAIL` / `NO_REPLY_EMAIL` secrets required). **Repository or Environment variables (optional):** @@ -154,21 +154,18 @@ The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job us | `HELM_APP_NAME` | `app.kubernetes.io/name` for `kubectl rollout status` | `github.event.repository.name` | | `DEPLOY_ROLLOUT_TIMEOUT` | Rollout wait | `10m` | | `DEPLOY_BOOTSTRAP_JOB_TIMEOUT` | Bootstrap Job wait | `25m` | -| `SMTP_ADDRESS` | Host for `mailhost` in values (e.g. SendGrid) | (use literals in `ops/*-deploy.tmpl.yaml` if not set) | -| `SMTP_USER_NAME` | `mailuser` (e.g. `apikey` for SendGrid) | | -| `SMTP_PORT` / `SOCKET_PORT` | Ports for JavaMail | workflow defaults `SMTP_PORT` to `25` if unset | -| `SMTP_AUTH` | Must be `true` for authenticated SMTP (injected by `envsubst` into `smtp_auth`) | | -| `SMTP_STARTTLS`, `SMTP_TYPE`, `SMTP_ENABLED`, `SMTP_DOMAIN` | Passed through to job env; if `NO_REPLY_EMAIL` is empty but `SMTP_DOMAIN` is set, the workflow sets `NO_REPLY_EMAIL=noreply@$SMTP_DOMAIN` before `envsubst` | | Default Helm **release** and **namespace** are **`-`** (e.g. `demo-dataverseup`). Override with workflow inputs `k8s_release_name` / `k8s_namespace` when needed. +For **demo**, SMTP host, ports, auth flags, and **`support@notch8.com`** addresses live only in **`ops/demo-deploy.tmpl.yaml`** — the workflow does **not** pass GitHub Variables for those; change the template (or add `${VAR}` placeholders and extend `envsubst`) if you need per-environment overrides. + **Migrating or renaming a release:** Update the literals in **`ops/-deploy.tmpl.yaml`** (ingress hosts, `dataverse_*` / `DATAVERSE_*` / `hostname`, `solrHttpBase`, `SOLR_*`, `DATAVERSE_URL`, `awsS3.bucketName`, DB names, etc.) so they match the new Helm release and namespace; then align Postgres, S3, TLS, and running workloads. #### SMTP (JavaMail) — enable, secrets, and how to test 1. **Chart:** set **`mail.enabled: true`** so **`010-mailrelay-set.sh`** is in the ConfigMap. Pod env must include the names the script reads: **`system_email`**, **`mailhost`**, **`mailuser`**, **`no_reply_email`**, **`smtp_password`**, **`smtp_port`**, **`socket_port`**, **`smtp_auth`**, **`smtp_starttls`**, **`smtp_type`**, **`smtp_enabled`** (see **`ops/demo-deploy.tmpl.yaml`** and **`scripts/init.d/010-mailrelay-set.sh`**). **`smtp_enabled`** must not be `false` / `0` / `no`, and **`system_email`** must be non-empty or the script no-ops. -2. **GitHub Environment (demo):** Add **Secrets** — `SMTP_PASSWORD` (SendGrid API key or provider password), `SYSTEM_EMAIL` (installation support address; `:SystemEmail`), `NO_REPLY_EMAIL` (JavaMail **From** / `no_reply_email`). Optionally set **Variables** (`SMTP_ADDRESS`, `SMTP_USER_NAME`, `SMTP_PORT`, `SMTP_AUTH=true`, etc.) if you do not want them as literals in the template. Redeploy so a **new pod** runs init (the script uses **`asadmin create-javamail-resource`** on boot). +2. **GitHub Environment (demo):** Add **Secret** **`SMTP_PASSWORD`** (SendGrid API key). **`system_email`** and **`no_reply_email`** are set to **`support@notch8.com`** in **`ops/demo-deploy.tmpl.yaml`**. Redeploy so a **new pod** runs init (the script uses **`asadmin create-javamail-resource`** on boot). 3. **Verify config:** After the pod is ready, check the setting: `curl -sS "https:///api/admin/settings/:SystemEmail"` (use a superuser API token if the endpoint requires auth on your version). In the UI, use **Contact** / support mail, **Forgot password**, or user signup (if enabled) and confirm delivery (and provider dashboard / spam folder). @@ -176,6 +173,18 @@ Default Helm **release** and **namespace** are **`--deploy.yaml` (CI artifact or run `envsubst` locally) and check `smtp_password` is non-empty (not the literal `${SMTP_PASSWORD}`) and `system_email` / `no_reply_email` are **`support@notch8.com`** for demo. + - **SendGrid:** use the **full API key** as `SMTP_PASSWORD`, **`mailuser` = `apikey`**, port **587**, **`smtp_auth` / `smtp_type`** as in **`ops/demo-deploy.tmpl.yaml`**. In [SendGrid Activity](https://app.sendgrid.com/email_activity), see bounces, blocks, and “unauthenticated” sends. **Verify** the **From** address (`no_reply_email`) via Single Sender or domain authentication. + - **Payara:** successful sends often log **nothing** at default levels. Exec into the Dataverse container and run: + `asadmin --user "$ADMIN_USER" --passwordfile "$PASSWORD_FILE" list-javamail-resources` + You should see **`mail/notifyMailSession`**. If it is missing, init did not configure mail (wrong env, `smtp_enabled`, or script failed). + **Domain logs (GDCC base image):** Payara lives under **`/opt/payara/appserver`**, not `/opt/payara/glassfish` alone. Typical file: + `/opt/payara/appserver/glassfish/domains/domain1/logs/server.log` + Or discover: `find /opt/payara -name server.log 2>/dev/null`. Much application output also goes to **`kubectl logs`** on the main container (`dataverseup`). + - **Network:** from the pod, `nc -zv smtp.sendgrid.net 587` (or your `mailhost`) must succeed if the cluster egress allows it. + ### Ingress and TLS Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 10843ba..5bf3f18 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -5,10 +5,11 @@ # They must stay consistent with your Helm **release name** and **namespace** (e.g. both `demo-dataverseup`). # # **envsubst** (CI: see `.github/workflows/deploy.yaml`) only injects secrets and the rollout nonce — not URLs: -# $DB_PASSWORD, $SYSTEM_EMAIL, $NO_REPLY_EMAIL, $SMTP_PASSWORD, $SMTP_AUTH, ${GITHUB_RUN_ID} +# $DB_PASSWORD, $SMTP_PASSWORD, ${GITHUB_RUN_ID} +# **system_email** / **no_reply_email** are literals here (both support@notch8.com for SendGrid From + :SystemEmail). # Local render: -# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SYSTEM_EMAIL=... NO_REPLY_EMAIL=... SMTP_PASSWORD=... SMTP_AUTH= -# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $NO_REPLY_EMAIL $SMTP_PASSWORD $SMTP_AUTH' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml +# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SMTP_PASSWORD=... +# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SMTP_PASSWORD' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml # # Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), # same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/DEPLOYMENT.md**). @@ -260,9 +261,7 @@ extraEnvVars: value: "true" - name: SMTP_DOMAIN value: "notch8.com" - # Script reads these names (aligned with SMTP_* above). - # no_reply_email → Payara JavaMail --fromaddress (outbound From). No separate Reply-To in the session; - # mail clients reply to From when Reply-To is absent, so use support@ for both behaviors. + # Script reads these names (aligned with SMTP_* above). Both support@notch8.com: :SystemEmail + JavaMail From. - name: system_email value: "support@notch8.com" - name: mailhost From e1a04cc5d71c56b2f038f506c874f3f78b4943b2 Mon Sep 17 00:00:00 2001 From: April Rieger Date: Sat, 4 Apr 2026 00:27:08 -0700 Subject: [PATCH 31/31] Update footer and add correct env vrs for smtp mail --- branding/branding.env | 2 +- docs/DEPLOYMENT.md | 8 +++++++- ops/demo-deploy.tmpl.yaml | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/branding/branding.env b/branding/branding.env index 3c576b8..4d97138 100644 --- a/branding/branding.env +++ b/branding/branding.env @@ -21,7 +21,7 @@ DISABLE_ROOT_DATAVERSE_THEME=true # Multi-word values must be quoted. FOOTER_COPYRIGHT is appended to the built-in "Copyright © YEAR" with no # separator in the UI; apply-branding.sh prepends a space unless you start with space/tab, —, -, |, or (. -FOOTER_COPYRIGHT=' DataverseUp | Powered by Notch8, your partner in digital preservation.' +FOOTER_COPYRIGHT=' DataverseUp | Hosted by Notch8, your partner in digital preservation.' NAVBAR_ABOUT_URL=https://notch8.com/about NAVBAR_SUPPORT_URL=mailto:support@notch8.com diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 3321e9a..0d7115e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -165,7 +165,7 @@ For **demo**, SMTP host, ports, auth flags, and **`support@notch8.com`** address 1. **Chart:** set **`mail.enabled: true`** so **`010-mailrelay-set.sh`** is in the ConfigMap. Pod env must include the names the script reads: **`system_email`**, **`mailhost`**, **`mailuser`**, **`no_reply_email`**, **`smtp_password`**, **`smtp_port`**, **`socket_port`**, **`smtp_auth`**, **`smtp_starttls`**, **`smtp_type`**, **`smtp_enabled`** (see **`ops/demo-deploy.tmpl.yaml`** and **`scripts/init.d/010-mailrelay-set.sh`**). **`smtp_enabled`** must not be `false` / `0` / `no`, and **`system_email`** must be non-empty or the script no-ops. -2. **GitHub Environment (demo):** Add **Secret** **`SMTP_PASSWORD`** (SendGrid API key). **`system_email`** and **`no_reply_email`** are set to **`support@notch8.com`** in **`ops/demo-deploy.tmpl.yaml`**. Redeploy so a **new pod** runs init (the script uses **`asadmin create-javamail-resource`** on boot). +2. **GitHub Environment (demo):** Add **Secret** **`SMTP_PASSWORD`** (SendGrid API key). **`ops/demo-deploy.tmpl.yaml`** sets **`DATAVERSE_MAIL_SYSTEM_EMAIL`** / **`DATAVERSE_MAIL_SUPPORT_EMAIL`** to **`support@notch8.com`** and **`DATAVERSE_MAIL_MTA_*`** for SendGrid (587 + STARTTLS + **`apikey`** user). Since **Dataverse 6.2+**, outbound mail uses these **MicroProfile** settings ([SMTP/Email in the Dataverse Guide](https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration)); **`010-mailrelay-set.sh`** (Payara JavaMail + `:SystemEmail`) is **not sufficient alone**. Set **`DATAVERSE_MAIL_DEBUG`** to **`true`** in the template temporarily for verbose mail logs ([`dataverse.mail.debug`](https://guides.dataverse.org/en/latest/installation/config.html)). **Restart pods** after changing mail env (MTA session is cached). 3. **Verify config:** After the pod is ready, check the setting: `curl -sS "https:///api/admin/settings/:SystemEmail"` (use a superuser API token if the endpoint requires auth on your version). In the UI, use **Contact** / support mail, **Forgot password**, or user signup (if enabled) and confirm delivery (and provider dashboard / spam folder). @@ -185,6 +185,12 @@ For **demo**, SMTP host, ports, auth flags, and **`support@notch8.com`** address Or discover: `find /opt/payara -name server.log 2>/dev/null`. Much application output also goes to **`kubectl logs`** on the main container (`dataverseup`). - **Network:** from the pod, `nc -zv smtp.sendgrid.net 587` (or your `mailhost`) must succeed if the cluster egress allows it. +7. **Why you don’t see Rails-style “Sent mail” lines:** **ActionMailer** logs each delivery at INFO by default. **Dataverse 6.2+** uses **`dataverse.mail.*`** / **`DATAVERSE_MAIL_*`**; with **`DATAVERSE_MAIL_DEBUG=true`** you get the **supported** verbose logging described in the [installation guide](https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration) (set in **`ops/demo-deploy.tmpl.yaml`**, then roll pods — **disable after debugging**). + + **Lower-level JavaMail trace (optional):** `-Dmail.debug=true` on **`JVM_OPTS`** still works for raw SMTP wire logs but can leak credentials; prefer **`DATAVERSE_MAIL_DEBUG`** first. + + **Closest thing to a delivery log:** **[SendGrid Activity](https://app.sendgrid.com/email_activity)** (accept / bounce / block). + ### Ingress and TLS Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 5bf3f18..75d5fa1 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -231,6 +231,27 @@ extraEnvVars: value: "demo-dataverseup" - name: JVM_OPTS value: "-Xmx2g -Xms2g" + # Dataverse 6.2+ sends mail via MicroProfile dataverse.mail.* (env DATAVERSE_MAIL_*), not only :SystemEmail / + # Payara JavaMail. Without these, verification/password mail may never send. See: + # https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration + - name: DATAVERSE_MAIL_SYSTEM_EMAIL + value: "support@notch8.com" + - name: DATAVERSE_MAIL_SUPPORT_EMAIL + value: "support@notch8.com" + - name: DATAVERSE_MAIL_DEBUG + value: "false" + - name: DATAVERSE_MAIL_MTA_HOST + value: "smtp.sendgrid.net" + - name: DATAVERSE_MAIL_MTA_PORT + value: "587" + - name: DATAVERSE_MAIL_MTA_AUTH + value: "true" + - name: DATAVERSE_MAIL_MTA_USER + value: "apikey" + - name: DATAVERSE_MAIL_MTA_PASSWORD + value: "${SMTP_PASSWORD}" + - name: DATAVERSE_MAIL_MTA_STARTTLS_ENABLE + value: "true" - name: DATAVERSE_PID_PROVIDERS value: demo - name: DATAVERSE_PID_DEFAULT_PROVIDER