diff --git a/.env.example b/.env.example index bc57ec4..eb8c5b2 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ DATAVERSE_DB_NAME=dataverse SOLR_SERVICE_HOST=solr SOLR_SERVICE_PORT=8983 -# Used by init.d scripts inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme. +# Used by scripts/init.d inside the Dataverse container (curl to localhost:8080). Host:port only, no URL scheme. DATAVERSE_URL=localhost:8080 DATAVERSE_SERVICE_HOST=localhost.direct # Optional: admin API token for branding/seed compose jobs (normally secrets/api/key is filled from bootstrap.env by dev_branding / dev_seed entrypoints). @@ -26,7 +26,7 @@ POSTGRES_DATABASE=dataverse POSTGRES_DB=dataverse POSTGRES_PORT=5432 -# Public hostname (init.d sets dataverse.siteUrl to https:// — use Stack Car *.localhost.direct). +# Public hostname (scripts/init.d sets dataverse.siteUrl to https:// — use Stack Car *.localhost.direct). hostname=localhost.direct # Traefik Host() rules use ${traefikhost} and subdomains (solr., minio., …). traefikhost=localhost.direct @@ -39,22 +39,35 @@ MINIO_ROOT_PASSWORD=love1234 # Optional; unused for local dev when TLS is provided by `sc proxy` (Stack Car). useremail=you@example.org +# --- Optional: SMTP (scripts/init.d/010-mailrelay-set.sh; requires system_email; smtp_enabled not false/0/no) --- +# smtp_enabled=true +# smtp_type=plain +# smtp_auth=true +# smtp_starttls=true +# smtp_port=587 +# socket_port=587 +# mailhost=smtp.example.com +# mailuser=apikey-or-username +# smtp_password=secret +# system_email=support@example.com +# no_reply_email=noreply@example.com + WEBHOOK=/opt/payara/triggers/external-services.py -# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / init.d) +# Optional: DataCite test (leave as-is for FAKE DOIs via bootstrap / scripts/init.d) dataciterestapiurlstring=https\\:\/\/api.test.datacite.org baseurlstring=https\:\/\/mds.test.datacite.org -# Optional builtin-user API key override (see init.d/01-persistent-id.sh) +# Optional builtin-user API key override (see scripts/init.d/01-persistent-id.sh) # BUILTIN_USERS_KEY=burrito -# --- Optional: MinIO buckets (only if init.d minio scripts should run) --- +# --- Optional: MinIO buckets (only if scripts/init.d minio scripts should run) --- # minio_label_1= # bucketname_1= # minio_bucket_1= # minio_profile_1= -# --- Optional: AWS S3 store (only if init.d/006-s3-aws-storage.sh should run) --- +# --- Optional: AWS S3 store (only if scripts/init.d/006-s3-aws-storage.sh should run) --- # aws_bucket_name= # aws_endpoint_url= # aws_s3_profile= diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 40e464a..fae6bb9 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -27,7 +27,7 @@ on: required: false type: string run_bootstrap_job: - description: Re-run the bootstrap Job (helmHook=false) — same manifest as chart templates/bootstrap-job.yaml; use if the post-install hook was skipped or you reset Postgres. Chart default mode is oneShot (configbaker bootstrap.sh dev). Deletes the prior Job by name first. + description: Re-run the bootstrap Job (helmHook=false, compose). Requires Secret dataverse-admin-api-token (key token) in the namespace when the instance is already bootstrapped — see ops/*-deploy.tmpl.yaml bootstrapJob.compose.existingAdminApiTokenSecret. Deletes the prior Job by name first. required: false type: boolean default: false @@ -42,20 +42,8 @@ jobs: environment: ${{ inputs.environment }} env: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} - # Optional mail — secrets + Environment variables (see ops/demo-deploy.tmpl.yaml header). - SYSTEM_EMAIL: ${{ secrets.SYSTEM_EMAIL }} - NO_REPLY_EMAIL: ${{ secrets.NO_REPLY_EMAIL }} + # Mail: SendGrid API key only — host/ports/From/support address are literals in ops/-deploy.tmpl.yaml. SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} - MAIL_SMTP_PASSWORD: ${{ secrets.MAIL_SMTP_PASSWORD }} - SMTP_ADDRESS: ${{ vars.SMTP_ADDRESS }} - SMTP_USER_NAME: ${{ vars.SMTP_USER_NAME }} - SMTP_PORT: ${{ vars.SMTP_PORT }} - SOCKET_PORT: ${{ vars.SOCKET_PORT }} - SMTP_AUTH: ${{ vars.SMTP_AUTH }} - SMTP_STARTTLS: ${{ vars.SMTP_STARTTLS }} - SMTP_TYPE: ${{ vars.SMTP_TYPE }} - SMTP_ENABLED: ${{ vars.SMTP_ENABLED }} - SMTP_DOMAIN: ${{ vars.SMTP_DOMAIN }} DEPLOY_ENVIRONMENT: ${{ inputs.environment }} HELM_RELEASE_NAME: ${{ inputs.k8s_release_name || format('{0}-{1}', inputs.environment, github.event.repository.name) }} HELM_NAMESPACE: ${{ inputs.k8s_namespace || format('{0}-{1}', inputs.environment, github.event.repository.name) }} @@ -94,16 +82,19 @@ jobs: TMPL="ops/${DEPLOY_ENVIRONMENT}-deploy.tmpl.yaml" OUT="ops/${DEPLOY_ENVIRONMENT}-deploy.yaml" echo "$KUBECONFIG_FILE" | base64 -d >"$KUBECONFIG" - export SMTP_PORT="${SMTP_PORT:-25}" - export SOCKET_PORT="${SOCKET_PORT:-${SMTP_PORT}}" - export SMTP_PASSWORD="${SMTP_PASSWORD:-${MAIL_SMTP_PASSWORD:-}}" - if [ -z "${NO_REPLY_EMAIL:-}" ] && [ -n "${SMTP_DOMAIN:-}" ]; then - export NO_REPLY_EMAIL="noreply@${SMTP_DOMAIN}" - fi - # Only secrets + rollout id — hosts, Solr DNS, ingress, and bucket are literals in the *.tmpl.yaml file. - ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SYSTEM_EMAIL $SMTP_PASSWORD $SMTP_AUTH' + # envsubst: only secrets + rollout id. Everything else (SMTP host, ports, From, URLs, Solr, …) is literals in the *.tmpl.yaml file. + ENVSUBST_VARS='$GITHUB_RUN_ID $DB_PASSWORD $SMTP_PASSWORD' envsubst "$ENVSUBST_VARS" <"$TMPL" >"$OUT" + - name: Solr conf ConfigMap (pre-Helm) + env: + DV_REF: ${{ vars.DV_REF || 'v6.10.1' }} + SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }} + run: | + set -e + chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh + SOLR_RESTART_DEPLOYMENTS=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME" + - name: Deploy with Helm run: | set -e @@ -116,6 +107,16 @@ jobs: -l "app.kubernetes.io/instance=${HELM_RELEASE_NAME},app.kubernetes.io/name=${HELM_APP_NAME}" \ --timeout="${DEPLOY_ROLLOUT_TIMEOUT}" + - name: Solr workloads pick up ConfigMap updates + if: ${{ vars.SOLR_POST_HELM_ROLLOUT == 'true' }} + env: + DV_REF: ${{ vars.DV_REF || 'v6.10.1' }} + SOLR_DIST_VERSION: ${{ vars.SOLR_DIST_VERSION || '9.10.1' }} + run: | + set -e + chmod +x scripts/solr-init-k8s.sh scripts/k8s/ensure-solr-conf-configmap.sh + SOLR_APPLY_CM=false ./scripts/solr-init-k8s.sh "$HELM_NAMESPACE" "$HELM_RELEASE_NAME" + - name: One-shot bootstrap Job (non-hook) if: github.event_name == 'workflow_dispatch' && inputs.run_bootstrap_job run: | @@ -125,9 +126,11 @@ jobs: helm template "$HELM_RELEASE_NAME" "$HELM_CHART_PATH" \ --namespace "$HELM_NAMESPACE" \ $HELM_EXTRA_ARGS \ + --show-only templates/bootstrap-chain-configmap.yaml \ --show-only templates/bootstrap-job.yaml \ --set bootstrapJob.enabled=true \ --set bootstrapJob.helmHook=false \ + --set bootstrapJob.mode=compose \ >"$OUT" JOB="$(awk '/^kind: Job$/{j=1} j && /^ name: /{print $2; exit}' "$OUT")" if [ -z "$JOB" ]; then @@ -137,4 +140,4 @@ jobs: kubectl -n "$HELM_NAMESPACE" delete job "$JOB" --ignore-not-found=true kubectl apply -f "$OUT" kubectl -n "$HELM_NAMESPACE" wait --for=condition=complete "job/$JOB" --timeout="${DEPLOY_BOOTSTRAP_JOB_TIMEOUT}" - kubectl -n "$HELM_NAMESPACE" logs "job/$JOB" + kubectl -n "$HELM_NAMESPACE" logs "job/$JOB" \ No newline at end of file diff --git a/.gitignore b/.gitignore index e134889..5b479ff 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ minio-data/ data/ /docroot/ *.war -.idea/ +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 28395ab..6b5e4d6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # DataverseUp -Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and room to grow toward hosted AWS/Kubernetes without forking core. +Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (GDCC container images), aligned with the **DataverseUp** plan: pinned versions, compose-first bring-up, and a **Helm chart** for Kubernetes without forking core. + +| | | +|--|--| +| **Repository** | [github.com/notch8/dataverseup](https://github.com/notch8/dataverseup) | +| **Live demo** | [demo-dataverseup.notch8.cloud](https://demo-dataverseup.notch8.cloud/) — deployed from this stack (Helm/Kubernetes); seeded demo data for smoke tests | ## Quick start (local / lab) @@ -46,6 +51,13 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G ``` If you need a manual token, put it on one line in **`secrets/api/key`** (superuser token from the UI), then run **`dev_branding`** again. +## Kubernetes (Helm) + +- **Chart:** `charts/dataverseup` — see **[charts/dataverseup/README.md](charts/dataverseup/README.md)** for a feature summary and `helm` commands. +- **Runbook:** **[docs/DEPLOYMENT.md](docs/DEPLOYMENT.md)** — Helm (prereqs, Secrets, Solr, S3, bootstrap), GitHub Actions deploy, `./bin/helm_deploy`, smoke tests, upgrades. + +Compose remains the default path for local/lab; Helm reuses the same Payara scripts from **`scripts/init.d/`** (chart symlinks under `charts/dataverseup/files/init.d/`) where applicable. + ## Layout | Path | Purpose | @@ -53,15 +65,14 @@ Notch8's **ops wrapper** around stock **[Dataverse](https://dataverse.org/)** (G | `docker-compose.yml` | Stack: Postgres, Solr, MinIO (optional), Dataverse, bootstrap, branding, seed; Traefik labels + **`networks.default.name: stackcar`** (Stack Car proxy) | | `.env.example` | Version pins and env template — copy to `.env` | | `secrets.example/` | Payara/Dataverse secret files template — copy to **`secrets/`** (see Quick start) | -| `init.d/` | Payara init scripts (local storage, optional S3/MinIO when env set) | -| `init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` | -| `config/schema.xml`, `config/solrconfig.xml` | Solr conf bind-mounts / upstream copies (see `scripts/solr-initdb/`) | -| `config/update-fields.sh` | Upstream metadata-block tooling helper | +| `scripts/` | **All automation:** Payara **`init.d/`**, Compose/K8s entrypoints, **`apply-branding.sh`**, **`seed-content.sh`**, **`solr-initdb/`**, **`solr/update-fields.sh`**, **`triggers/`**, **`k8s/`** | +| `scripts/init.d/vendor-solr/` | Vendored Solr helpers for `1002-custom-metadata.sh` | +| `config/schema.xml`, `config/solrconfig.xml` | Solr XML bind-mounts / upstream copies (see `scripts/solr-initdb/`) | | `branding/` | Installation branding + static assets | | `fixtures/seed/` | JSON + files for **`dev_seed`** | -| `scripts/` | Bootstrap, branding, seed entrypoints, `apply-branding.sh`, `solr-initdb/` | -| `triggers/` | Postgres notify + optional webhook script (see **`WEBHOOK`** in `.env.example`) | -| `docs/DEPLOYMENT.md` | **Working deployment notes + learnings** (add in-repo when you maintain runbooks) | +| `charts/dataverseup/` | Helm chart for Dataverse on Kubernetes (optional Solr, bootstrap Job, S3, Ingress, …) | +| `bin/helm_deploy` | Wrapper around `helm upgrade --install` with sane defaults (see **`docs/DEPLOYMENT.md`**) | +| `docs/DEPLOYMENT.md` | **Deployment runbook:** Helm/Kubernetes, optional GitHub Actions, Compose pointer, learnings log | ## Version pin @@ -73,8 +84,8 @@ Compose uses **`solr:9.10.1`** with IQSS **`schema.xml`** / **`solrconfig.xml`** REF=develop # or a release tag, e.g. v6.10.1 curl -fsSL -o config/schema.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/schema.xml" curl -fsSL -o config/solrconfig.xml "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/solrconfig.xml" -curl -fsSL -o config/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh" -chmod +x config/update-fields.sh +curl -fsSL -o scripts/solr/update-fields.sh "https://raw.githubusercontent.com/IQSS/dataverse/${REF}/conf/solr/update-fields.sh" +chmod +x scripts/solr/update-fields.sh ``` If you previously ran Solr 8, remove the compose Solr volume once so the core is recreated under Solr 9, then reindex from Dataverse. @@ -88,4 +99,4 @@ If you previously ran Solr 8, remove the compose Solr volume once so the core is ## License -Dataverse is licensed by IQSS; container images by their publishers. +Dataverse is licensed by IQSS; container images by their publishers. \ No newline at end of file diff --git a/bin/helm_deploy b/bin/helm_deploy new file mode 100755 index 0000000..ca149ff --- /dev/null +++ b/bin/helm_deploy @@ -0,0 +1,30 @@ +#!/bin/sh +# +# Helm upgrade/install for the Dataverse chart (from repository root). +# Chart path defaults to ./charts/dataverseup; override with HELM_CHART_PATH. +# Pass extra flags via HELM_EXTRA_ARGS, e.g.: +# HELM_EXTRA_ARGS="--values ./my-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace +# +# A second --timeout in HELM_EXTRA_ARGS overrides the default below (Helm uses the last value). + +set -e + +if [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + echo "Usage: ./bin/helm_deploy RELEASE_NAME NAMESPACE" >&2 + echo " Run from the repository root (HELM_CHART_PATH defaults to ./charts/dataverseup)." >&2 + exit 1 +fi + +release_name="$1" +namespace="$2" +chart_path="${HELM_CHART_PATH:-./charts/dataverseup}" + +helm upgrade \ + --install \ + --atomic \ + --timeout 30m0s \ + ${HELM_EXTRA_ARGS:-} \ + --namespace="$namespace" \ + --create-namespace \ + "$release_name" \ + "$chart_path" diff --git a/branding.env b/branding.env new file mode 120000 index 0000000..e648920 --- /dev/null +++ b/branding.env @@ -0,0 +1 @@ +branding/branding.env \ No newline at end of file diff --git a/branding/branding.env b/branding/branding.env index 47f207c..4d97138 100644 --- a/branding/branding.env +++ b/branding/branding.env @@ -3,7 +3,7 @@ # For :HeaderCustomizationFile / :FooterCustomizationFile / :StyleCustomizationFile use *filesystem* paths in the # API, e.g. /dv/docroot/branding/custom-header.html (see branding.env examples below). -# INSTALLATION_NAME='My Repository' +INSTALLATION_NAME='DataverseUp' # Navbar logo file on disk: branding/docroot/logos/navbar/logo.svg → URL below (must match filename). LOGO_CUSTOMIZATION_FILE=/logos/navbar/logo.svg @@ -21,8 +21,8 @@ DISABLE_ROOT_DATAVERSE_THEME=true # Multi-word values must be quoted. FOOTER_COPYRIGHT is appended to the built-in "Copyright © YEAR" with no # separator in the UI; apply-branding.sh prepends a space unless you start with space/tab, —, -, |, or (. -FOOTER_COPYRIGHT='Notch8' -# NAVBAR_ABOUT_URL=https://example.edu/about -# NAVBAR_SUPPORT_URL=mailto:support@example.edu -# NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/ +FOOTER_COPYRIGHT=' DataverseUp | Hosted by Notch8, your partner in digital preservation.' +NAVBAR_ABOUT_URL=https://notch8.com/about +NAVBAR_SUPPORT_URL=mailto:support@notch8.com +NAVBAR_GUIDES_URL=https://guides.dataverse.org/en/latest/ diff --git a/charts/dataverseup/.helmignore b/charts/dataverseup/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/dataverseup/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/dataverseup/Chart.yaml b/charts/dataverseup/Chart.yaml new file mode 100644 index 0000000..3253651 --- /dev/null +++ b/charts/dataverseup/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: dataverseup +description: Dataverse (Payara) on Kubernetes; optional in-cluster standalone Solr; Postgres external; optional S3. + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.12 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "6.10.1-noble-r0" diff --git a/charts/dataverseup/README.md b/charts/dataverseup/README.md new file mode 100644 index 0000000..701d87c --- /dev/null +++ b/charts/dataverseup/README.md @@ -0,0 +1,38 @@ +# dataverseup Helm chart + +Deploys **stock Dataverse** (GDCC `gdcc/dataverse` image) on Kubernetes with optional: + +- **Persistent** file store and optional **docroot** PVC (branding / logos) +- **Bootstrap** Job (`gdcc/configbaker`) +- **Internal Solr** + **solrInit** initContainer +- **S3** storage driver (AWS-style credentials Secret) +- **Ingress**, **HPA**, **ServiceAccount** + +## Quick commands + +```bash +helm lint charts/dataverseup +helm template release-name charts/dataverseup -f your-values.yaml +helm upgrade --install release-name charts/dataverseup -n your-namespace -f your-values.yaml +``` + +## Documentation + +See **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)** in this repository for prerequisites, Secret layout, smoke tests, and CI deploy notes. + +## Payara init scripts (S3, mail) + +S3 and mail relay scripts (and the full Payara bundle when `initdFromChart.enabled`) live under **`files/init.d/`** as **symbolic links** to **`scripts/init.d/`** (same tree Docker Compose mounts at `/opt/payara/init.d`). Helm follows them when rendering and **`helm package` inlines their contents** into the chart archive, so published charts stay self-contained. + +## Configuration + +| Key | Purpose | +|-----|---------| +| `image.repository` / `image.tag` | GDCC Dataverse image | +| `extraEnvFrom` / `extraEnvVars` | DB, Solr, URL, JVM — **use Secrets for credentials** | +| `persistence` | RWO PVC for `/data` | +| `internalSolr` + `solrInit` | **Dedicated Solr 9** pod with this release (not a shared cluster Solr). Default **`solrInit.mode: standalone`**; empty **`solrInit.solrHttpBase`** → chart uses the in-release Solr Service. Core **`dataverse`** (Compose uses **`collection1`**). SolrCloud (`mode: cloud`) needs ZK + **full** conf or `solr-conf.tgz`. | +| `bootstrapJob` | First-time `configbaker` bootstrap | +| `ingress` | HTTP routing to Service port 80 | + +Solr alignment with Docker Compose (IQSS **`config/`** files, core naming, `solrInit` overrides) is documented in **[docs/DEPLOYMENT.md](../../docs/DEPLOYMENT.md)**. diff --git a/charts/dataverseup/files/apply-branding.sh b/charts/dataverseup/files/apply-branding.sh new file mode 120000 index 0000000..955c071 --- /dev/null +++ b/charts/dataverseup/files/apply-branding.sh @@ -0,0 +1 @@ +../../../scripts/apply-branding.sh \ No newline at end of file diff --git a/charts/dataverseup/files/branding-navbar-logo.svg b/charts/dataverseup/files/branding-navbar-logo.svg new file mode 120000 index 0000000..800438c --- /dev/null +++ b/charts/dataverseup/files/branding-navbar-logo.svg @@ -0,0 +1 @@ +../../../branding/docroot/logos/navbar/logo.svg \ No newline at end of file diff --git a/charts/dataverseup/files/branding.env b/charts/dataverseup/files/branding.env new file mode 120000 index 0000000..49f62f1 --- /dev/null +++ b/charts/dataverseup/files/branding.env @@ -0,0 +1 @@ +../../../branding.env \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/dataset-images.json b/charts/dataverseup/files/fixtures-seed/dataset-images.json new file mode 120000 index 0000000..a2959e6 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/dataset-images.json @@ -0,0 +1 @@ +../../../../fixtures/seed/dataset-images.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/dataset-tabular.json b/charts/dataverseup/files/fixtures-seed/dataset-tabular.json new file mode 120000 index 0000000..a4feaa4 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/dataset-tabular.json @@ -0,0 +1 @@ +../../../../fixtures/seed/dataset-tabular.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/demo-collection.json b/charts/dataverseup/files/fixtures-seed/demo-collection.json new file mode 120000 index 0000000..dd0f9c3 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/demo-collection.json @@ -0,0 +1 @@ +../../../../fixtures/seed/demo-collection.json \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_1x1.png b/charts/dataverseup/files/fixtures-seed/files_1x1.png new file mode 120000 index 0000000..cb45e4a --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_1x1.png @@ -0,0 +1 @@ +../../../../fixtures/seed/files/1x1.png \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_badge.svg b/charts/dataverseup/files/fixtures-seed/files_badge.svg new file mode 120000 index 0000000..b63ec20 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_badge.svg @@ -0,0 +1 @@ +../../../../fixtures/seed/files/badge.svg \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_readme.txt b/charts/dataverseup/files/fixtures-seed/files_readme.txt new file mode 120000 index 0000000..c873320 --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_readme.txt @@ -0,0 +1 @@ +../../../../fixtures/seed/files/readme.txt \ No newline at end of file diff --git a/charts/dataverseup/files/fixtures-seed/files_sample.csv b/charts/dataverseup/files/fixtures-seed/files_sample.csv new file mode 120000 index 0000000..49e1a9f --- /dev/null +++ b/charts/dataverseup/files/fixtures-seed/files_sample.csv @@ -0,0 +1 @@ +../../../../fixtures/seed/files/sample.csv \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/006-s3-aws-storage.sh b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh new file mode 120000 index 0000000..8ef49dd --- /dev/null +++ b/charts/dataverseup/files/init.d/006-s3-aws-storage.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/006-s3-aws-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/01-persistent-id.sh b/charts/dataverseup/files/init.d/01-persistent-id.sh new file mode 120000 index 0000000..8c436eb --- /dev/null +++ b/charts/dataverseup/files/init.d/01-persistent-id.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/01-persistent-id.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-languages.sh b/charts/dataverseup/files/init.d/010-languages.sh new file mode 120000 index 0000000..b4b5fbb --- /dev/null +++ b/charts/dataverseup/files/init.d/010-languages.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/010-languages.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/010-mailrelay-set.sh b/charts/dataverseup/files/init.d/010-mailrelay-set.sh new file mode 120000 index 0000000..7830b52 --- /dev/null +++ b/charts/dataverseup/files/init.d/010-mailrelay-set.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/010-mailrelay-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/011-local-storage.sh b/charts/dataverseup/files/init.d/011-local-storage.sh new file mode 120000 index 0000000..59c1676 --- /dev/null +++ b/charts/dataverseup/files/init.d/011-local-storage.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/011-local-storage.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/012-minio-bucket1.sh b/charts/dataverseup/files/init.d/012-minio-bucket1.sh new file mode 120000 index 0000000..e4f3db6 --- /dev/null +++ b/charts/dataverseup/files/init.d/012-minio-bucket1.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/012-minio-bucket1.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/013-minio-bucket2.sh b/charts/dataverseup/files/init.d/013-minio-bucket2.sh new file mode 120000 index 0000000..fe3c9d8 --- /dev/null +++ b/charts/dataverseup/files/init.d/013-minio-bucket2.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/013-minio-bucket2.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/02-controlled-voc.sh b/charts/dataverseup/files/init.d/02-controlled-voc.sh new file mode 120000 index 0000000..aaa29e8 --- /dev/null +++ b/charts/dataverseup/files/init.d/02-controlled-voc.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/02-controlled-voc.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/03-doi-set.sh b/charts/dataverseup/files/init.d/03-doi-set.sh new file mode 120000 index 0000000..59f2e33 --- /dev/null +++ b/charts/dataverseup/files/init.d/03-doi-set.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/03-doi-set.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/04-setdomain.sh b/charts/dataverseup/files/init.d/04-setdomain.sh new file mode 120000 index 0000000..da0ff7a --- /dev/null +++ b/charts/dataverseup/files/init.d/04-setdomain.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/04-setdomain.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/05-reindex.sh b/charts/dataverseup/files/init.d/05-reindex.sh new file mode 120000 index 0000000..0e0087a --- /dev/null +++ b/charts/dataverseup/files/init.d/05-reindex.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/05-reindex.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/07-previewers.sh b/charts/dataverseup/files/init.d/07-previewers.sh new file mode 120000 index 0000000..d1867a7 --- /dev/null +++ b/charts/dataverseup/files/init.d/07-previewers.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/07-previewers.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/08-federated-login.sh b/charts/dataverseup/files/init.d/08-federated-login.sh new file mode 120000 index 0000000..f9e9344 --- /dev/null +++ b/charts/dataverseup/files/init.d/08-federated-login.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/08-federated-login.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1001-webhooks.sh b/charts/dataverseup/files/init.d/1001-webhooks.sh new file mode 120000 index 0000000..495712a --- /dev/null +++ b/charts/dataverseup/files/init.d/1001-webhooks.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/1001-webhooks.sh \ No newline at end of file diff --git a/charts/dataverseup/files/init.d/1002-custom-metadata.sh b/charts/dataverseup/files/init.d/1002-custom-metadata.sh new file mode 120000 index 0000000..479c9b6 --- /dev/null +++ b/charts/dataverseup/files/init.d/1002-custom-metadata.sh @@ -0,0 +1 @@ +../../../../scripts/init.d/1002-custom-metadata.sh \ No newline at end of file diff --git a/charts/dataverseup/files/k8s-bootstrap-chain.sh b/charts/dataverseup/files/k8s-bootstrap-chain.sh new file mode 120000 index 0000000..d01f1a7 --- /dev/null +++ b/charts/dataverseup/files/k8s-bootstrap-chain.sh @@ -0,0 +1 @@ +../../../scripts/k8s-bootstrap-chain.sh \ No newline at end of file diff --git a/charts/dataverseup/files/seed-content.sh b/charts/dataverseup/files/seed-content.sh new file mode 120000 index 0000000..17c06cc --- /dev/null +++ b/charts/dataverseup/files/seed-content.sh @@ -0,0 +1 @@ +../../../scripts/seed-content.sh \ No newline at end of file diff --git a/charts/dataverseup/templates/NOTES.txt b/charts/dataverseup/templates/NOTES.txt new file mode 100644 index 0000000..b69f582 --- /dev/null +++ b/charts/dataverseup/templates/NOTES.txt @@ -0,0 +1,29 @@ +{{- if .Values.internalSolr.enabled }} +Dedicated Solr with this release (not a shared cluster Solr): Service {{ include "dataverseup.fullname" . }}-solr port 8983 — in-namespace URL http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 . If solrInit.solrHttpBase is unset, the chart uses that URL for the init wait. Set app env (e.g. DATAVERSE_SOLR_HOST / SOLR_LOCATION) to the same host:port and core {{ .Values.solrInit.collection }}. + +{{- end }} +{{- if .Values.docrootPersistence.enabled }} +Docroot / branding: PVC {{ if .Values.docrootPersistence.existingClaim }}{{ .Values.docrootPersistence.existingClaim }}{{ else }}{{ include "dataverseup.fullname" . }}-docroot{{ end }} mounted at {{ .Values.docrootPersistence.mountPath }} (DATAVERSE_FILES_DOCROOT for /logos/*). +{{- end }} +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "dataverseup.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "dataverseup.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "dataverseup.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "dataverseup.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=primary" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/dataverseup/templates/_helpers.tpl b/charts/dataverseup/templates/_helpers.tpl new file mode 100644 index 0000000..61edcda --- /dev/null +++ b/charts/dataverseup/templates/_helpers.tpl @@ -0,0 +1,120 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "dataverseup.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dataverseup.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dataverseup.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dataverseup.labels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +{{ include "dataverseup.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dataverseup.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dataverseup.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Bootstrap / configbaker Job pod labels. Must NOT use selectorLabels alone: Deployment matchLabels are only +name+instance, so Job pods would match and `kubectl logs deploy/` can pick the wrong pod. +*/}} +{{- define "dataverseup.bootstrapPodLabels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-bootstrap +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Service / CLI label query for the main Dataverse pods only. Pods also set component=primary; +Deployment matchLabels stay name+instance only so upgrades do not hit immutable selector changes. +*/}} +{{- define "dataverseup.primarySelectorLabels" -}} +{{ include "dataverseup.selectorLabels" . }} +app.kubernetes.io/component: primary +{{- end }} + +{{/* +Labels for the optional in-chart standalone Solr Deployment/Service (must NOT match dataverseup.selectorLabels +or the main Deployment ReplicaSet will count Solr pods). +*/}} +{{- define "dataverseup.internalSolrLabels" -}} +helm.sh/chart: {{ include "dataverseup.chart" . }} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-solr +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: internal-solr +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "dataverseup.internalSolrSelectorLabels" -}} +app.kubernetes.io/name: {{ include "dataverseup.name" . }}-solr +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: internal-solr +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "dataverseup.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "dataverseup.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Solr admin base URL (no path) for solrInit initContainer: explicit solrInit.solrHttpBase, else in-release Solr +Service when internalSolr is enabled, else a placeholder for external / shared cluster Solr (override required). +*/}} +{{- define "dataverseup.solrHttpBase" -}} +{{- if .Values.solrInit.solrHttpBase -}} +{{- .Values.solrInit.solrHttpBase -}} +{{- else if .Values.internalSolr.enabled -}} +http://{{ include "dataverseup.fullname" . }}-solr.{{ .Release.Namespace }}.svc.cluster.local:8983 +{{- else -}} +http://solr.solr.svc.cluster.local:8983 +{{- end -}} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-chain-configmap.yaml b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml new file mode 100644 index 0000000..4874f01 --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-chain-configmap.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.bootstrapJob.enabled (eq .Values.bootstrapJob.mode "compose") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: bootstrap-chain +binaryData: + files_1x1.png: {{ .Files.Get "files/fixtures-seed/files_1x1.png" | b64enc }} +data: + bootstrap-chain.sh: | +{{ .Files.Get "files/k8s-bootstrap-chain.sh" | trim | nindent 4 }} + apply-branding.sh: | +{{ .Files.Get "files/apply-branding.sh" | trim | nindent 4 }} + seed-content.sh: | +{{ .Files.Get "files/seed-content.sh" | trim | nindent 4 }} + branding.env: | +{{ .Files.Get "files/branding.env" | trim | nindent 4 }} + demo-collection.json: | +{{ .Files.Get "files/fixtures-seed/demo-collection.json" | trim | nindent 4 }} + dataset-images.json: | +{{ .Files.Get "files/fixtures-seed/dataset-images.json" | trim | nindent 4 }} + dataset-tabular.json: | +{{ .Files.Get "files/fixtures-seed/dataset-tabular.json" | trim | nindent 4 }} + files_badge.svg: | +{{ .Files.Get "files/fixtures-seed/files_badge.svg" | trim | nindent 4 }} + files_readme.txt: | +{{ .Files.Get "files/fixtures-seed/files_readme.txt" | trim | nindent 4 }} + files_sample.csv: | +{{ .Files.Get "files/fixtures-seed/files_sample.csv" | trim | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml new file mode 100644 index 0000000..f2e69dd --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-compose-postupgrade-job.yaml @@ -0,0 +1,115 @@ +{{- if and .Values.bootstrapJob.enabled (eq .Values.bootstrapJob.mode "compose") .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob.enabled .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob.existingSecret }} +{{- $pu := .Values.bootstrapJob.compose.postUpgradeBrandingSeedJob }} +{{- $dvUrl := .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap-branding-seed + namespace: {{ .Release.Namespace }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: configbaker-bootstrap-branding-seed + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-weight: "20" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +spec: + backoffLimit: {{ .Values.bootstrapJob.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.bootstrapJob.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "dataverseup.bootstrapPodLabels" . | nindent 8 }} + app.kubernetes.io/component: configbaker-bootstrap-branding-seed + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: configbaker + image: "{{ .Values.bootstrapJob.image.repository }}:{{ .Values.bootstrapJob.image.tag | default .Values.image.tag }}" + imagePullPolicy: {{ .Values.bootstrapJob.image.pullPolicy }} + command: ["/bin/bash", "/bootstrap-chain/bootstrap-chain.sh"] + env: + - name: BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP + value: "1" + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_INTERNAL_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_BOOTSTRAP_SECRETS_DIR + value: "/work" + - name: DATAVERSE_BOOTSTRAP_ENV_FILE + value: "/work/api/bootstrap.env" + - name: BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS + value: {{ .Values.bootstrapJob.compose.waitMaxSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_WAIT_SLEEP + value: {{ .Values.bootstrapJob.compose.waitSleepSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_SEED + value: {{ if .Values.bootstrapJob.compose.seed }}"1"{{ else }}"0"{{ end }} + - name: BRANDING_ENV_PATH + value: "/config/branding.env" + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + - name: DATAVERSE_API_TOKEN + valueFrom: + secretKeyRef: + name: {{ $pu.existingSecret | quote }} + key: {{ $pu.secretKey | quote }} + volumeMounts: + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true + {{- with .Values.bootstrapJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} +{{- end }} diff --git a/charts/dataverseup/templates/bootstrap-job.yaml b/charts/dataverseup/templates/bootstrap-job.yaml new file mode 100644 index 0000000..e3f989a --- /dev/null +++ b/charts/dataverseup/templates/bootstrap-job.yaml @@ -0,0 +1,128 @@ +{{- if .Values.bootstrapJob.enabled }} +{{- $dvUrl := .Values.bootstrapJob.dataverseUrl | default (printf "http://%s.%s.svc.cluster.local:%v" (include "dataverseup.fullname" .) .Release.Namespace .Values.service.port) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dataverseup.fullname" . }}-bootstrap{{ if not .Values.bootstrapJob.helmHook }}-once{{ end }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: configbaker-bootstrap + {{- if .Values.bootstrapJob.helmHook }} + annotations: + helm.sh/hook: post-install + helm.sh/hook-weight: "15" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed + {{- end }} +spec: + backoffLimit: {{ .Values.bootstrapJob.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.bootstrapJob.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "dataverseup.bootstrapPodLabels" . | nindent 8 }} + app.kubernetes.io/component: configbaker-bootstrap + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: configbaker + image: "{{ .Values.bootstrapJob.image.repository }}:{{ .Values.bootstrapJob.image.tag | default .Values.image.tag }}" + imagePullPolicy: {{ .Values.bootstrapJob.image.pullPolicy }} + {{- if eq .Values.bootstrapJob.mode "compose" }} + command: ["/bin/bash", "/bootstrap-chain/bootstrap-chain.sh"] + env: + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_INTERNAL_URL + value: {{ $dvUrl | quote }} + - name: DATAVERSE_BOOTSTRAP_SECRETS_DIR + value: "/work" + - name: DATAVERSE_BOOTSTRAP_ENV_FILE + value: "/work/api/bootstrap.env" + - name: BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS + value: {{ .Values.bootstrapJob.compose.waitMaxSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_WAIT_SLEEP + value: {{ .Values.bootstrapJob.compose.waitSleepSeconds | toString | quote }} + - name: BOOTSTRAP_CHAIN_SEED + value: {{ if .Values.bootstrapJob.compose.seed }}"1"{{ else }}"0"{{ end }} + - name: BRANDING_ENV_PATH + value: "/config/branding.env" + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + {{- if .Values.bootstrapJob.compose.existingAdminApiTokenSecret }} + - name: DATAVERSE_API_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.bootstrapJob.compose.existingAdminApiTokenSecret | quote }} + key: {{ .Values.bootstrapJob.compose.adminApiTokenSecretKey | default "token" | quote }} + optional: true + {{- end }} + volumeMounts: + - name: bootstrap-scripts + mountPath: /bootstrap-chain + readOnly: true + - name: bootstrap-work + mountPath: /work + - name: branding-env + mountPath: /config + readOnly: true + - name: seed-flat + mountPath: /seed-flat + readOnly: true + {{- else }} + command: {{- toYaml .Values.bootstrapJob.command | nindent 12 }} + env: + - name: DATAVERSE_URL + value: {{ $dvUrl | quote }} + - name: TIMEOUT + value: {{ .Values.bootstrapJob.timeout | quote }} + {{- end }} + {{- with .Values.bootstrapJob.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if eq .Values.bootstrapJob.mode "compose" }} + volumes: + - name: bootstrap-scripts + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + defaultMode: 0555 + items: + - key: bootstrap-chain.sh + path: bootstrap-chain.sh + - key: apply-branding.sh + path: apply-branding.sh + - key: seed-content.sh + path: seed-content.sh + - name: branding-env + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: branding.env + path: branding.env + - name: seed-flat + configMap: + name: {{ include "dataverseup.fullname" . }}-bootstrap-chain + items: + - key: demo-collection.json + path: demo-collection.json + - key: dataset-images.json + path: dataset-images.json + - key: dataset-tabular.json + path: dataset-tabular.json + - key: files_1x1.png + path: files_1x1.png + - key: files_badge.svg + path: files_badge.svg + - key: files_readme.txt + path: files_readme.txt + - key: files_sample.csv + path: files_sample.csv + - name: bootstrap-work + emptyDir: {} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/branding-navbar-configmap.yaml b/charts/dataverseup/templates/branding-navbar-configmap.yaml new file mode 100644 index 0000000..882d284 --- /dev/null +++ b/charts/dataverseup/templates/branding-navbar-configmap.yaml @@ -0,0 +1,12 @@ +{{- if .Values.brandingNavbarLogos.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-branding-navbar + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: branding-navbar +data: + logo.svg: | +{{ .Files.Get "files/branding-navbar-logo.svg" | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/configmap.yaml b/charts/dataverseup/templates/configmap.yaml new file mode 100644 index 0000000..375deb2 --- /dev/null +++ b/charts/dataverseup/templates/configmap.yaml @@ -0,0 +1,32 @@ +{{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-config + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +data: + {{- if .Values.initdFromChart.enabled }} + {{- range $path, $bytes := .Files.Glob "files/init.d/*.sh" }} + {{ base $path }}: | +{{- $bytes | toString | trim | nindent 4 }} + {{- end }} + {{- if .Values.configMap.enabled }} + {{- toYaml .Values.configMap.data | nindent 2 }} + {{- end }} + {{- else }} + {{- if .Values.configMap.enabled }} + {{- toYaml .Values.configMap.data | nindent 2 }} + {{- end }} + {{- if .Values.mail.enabled }} + {{- $mailScript := .Files.Get "files/init.d/010-mailrelay-set.sh" | trim }} + 010-mailrelay-set.sh: | +{{- $mailScript | nindent 4 }} + {{- end }} + {{- if .Values.awsS3.enabled }} + {{- $s3Script := .Files.Get "files/init.d/006-s3-aws-storage.sh" | trim }} + 006-s3-aws-storage.sh: | +{{- $s3Script | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/deployment.yaml b/charts/dataverseup/templates/deployment.yaml new file mode 100644 index 0000000..909b996 --- /dev/null +++ b/charts/dataverseup/templates/deployment.yaml @@ -0,0 +1,282 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "dataverseup.selectorLabels" . | nindent 6 }} + {{- with .Values.deploymentStrategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "dataverseup.labels" . | nindent 8 }} + app.kubernetes.io/component: primary + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "dataverseup.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if or .Values.solrInit.enabled .Values.solrPreSetupInitContainer .Values.brandingNavbarLogos.enabled }} + initContainers: + {{- with .Values.solrPreSetupInitContainer }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.solrInit.enabled }} + {{- if and .Values.solrInit.existingSecret .Values.solrInit.adminUser }}{{ fail "solrInit: use either existingSecret or adminUser/adminPassword, not both" }}{{- end }} + - name: load-solr-config + image: "{{ .Values.solrInit.image }}" + imagePullPolicy: {{ .Values.solrInit.imagePullPolicy }} + securityContext: + {{- toYaml .Values.solrInit.securityContext | nindent 12 }} + command: ["/bin/bash", "/scripts/solr-init.sh"] + {{- if .Values.solrInit.existingSecret }} + envFrom: + - secretRef: + name: {{ .Values.solrInit.existingSecret | quote }} + {{- end }} + env: + {{- if or .Values.internalSolr.enabled (eq .Values.solrInit.mode "standalone") }} + - name: SOLR_INIT_MODE + value: "standalone" + {{- end }} + - name: SOLR_ZK_CONNECT + value: {{ .Values.solrInit.zkConnect | default "" | quote }} + - name: SOLR_HTTP_BASE + value: {{ include "dataverseup.solrHttpBase" . | quote }} + - name: SOLR_COLLECTION + value: {{ .Values.solrInit.collection | quote }} + - name: SOLR_CONFIGSET_NAME + value: {{ .Values.solrInit.configSetName | quote }} + - name: SOLR_NUM_SHARDS + value: {{ .Values.solrInit.numShards | toString | quote }} + - name: SOLR_REPLICATION_FACTOR + value: {{ .Values.solrInit.replicationFactor | toString | quote }} + - name: SOLR_CONF_DIR + value: "/solr-conf" + - name: SOLR_BIN + value: {{ .Values.solrInit.solrBin | quote }} + {{- if and (not .Values.solrInit.existingSecret) .Values.solrInit.adminUser }} + - name: SOLR_ADMIN_USER + value: {{ .Values.solrInit.adminUser | quote }} + - name: SOLR_ADMIN_PASSWORD + value: {{ .Values.solrInit.adminPassword | quote }} + {{- end }} + volumeMounts: + - name: solr-init-script + mountPath: /scripts + readOnly: true + - name: solr-dataverse-conf + mountPath: /solr-conf + readOnly: true + {{- with .Values.solrInit.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.brandingNavbarLogos.enabled }} + - name: branding-docroot-init + image: {{ .Values.brandingNavbarLogos.initImage | quote }} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + DOCROOT='{{ ternary .Values.docrootPersistence.mountPath "/dv/docroot" .Values.docrootPersistence.enabled }}' + mkdir -p "${DOCROOT}/logos/navbar" + cp /brand/logo.svg "${DOCROOT}/logos/navbar/logo.svg" + chown -R {{ .Values.brandingNavbarLogos.docrootUid }}:{{ .Values.brandingNavbarLogos.docrootGid }} "${DOCROOT}" + volumeMounts: + - name: {{ if .Values.docrootPersistence.enabled }}docroot{{ else }}branding-writable-docroot{{ end }} + mountPath: {{ ternary .Values.docrootPersistence.mountPath "/dv/docroot" .Values.docrootPersistence.enabled | quote }} + - name: branding-navbar-cm + mountPath: /brand + readOnly: true + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.container.port }} + protocol: TCP + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or .Values.internalSolr.enabled .Values.extraEnvVars .Values.awsS3.enabled }} + env: + {{- if .Values.internalSolr.enabled }} + {{- $solrSvc := printf "%s-solr" (include "dataverseup.fullname" .) }} + # GDCC ct profile defaults to solr:8983 + core collection1; override for in-chart Solr (core from solrInit.collection). + - name: DATAVERSE_SOLR_HOST + value: {{ $solrSvc | quote }} + - name: DATAVERSE_SOLR_PORT + value: "8983" + - name: DATAVERSE_SOLR_CORE + value: {{ .Values.solrInit.collection | quote }} + - name: SOLR_SERVICE_HOST + value: {{ $solrSvc | quote }} + - name: SOLR_SERVICE_PORT + value: "8983" + - name: SOLR_LOCATION + value: {{ printf "%s:8983" $solrSvc | quote }} + {{- end }} + {{- if .Values.extraEnvVars }} + {{- toYaml .Values.extraEnvVars | nindent 12 }} + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: aws_endpoint_url + value: {{ .Values.awsS3.endpointUrl | quote }} + - name: aws_bucket_name + value: {{ .Values.awsS3.bucketName | quote }} + - name: aws_s3_profile + value: {{ .Values.awsS3.profile | quote }} + - name: aws_s3_region + value: {{ .Values.awsS3.region | quote }} + # Force correct SigV4 region (overrides a bad `region` line in the mounted config Secret). + - name: AWS_REGION + value: {{ .Values.awsS3.region | quote }} + - name: AWS_DEFAULT_REGION + value: {{ .Values.awsS3.region | quote }} + # Non-root Payara user cannot write /root/.aws; SDK reads these paths (mounted Secret). + - name: AWS_SHARED_CREDENTIALS_FILE + value: /secrets/aws-cli/.aws/credentials + - name: AWS_CONFIG_FILE + value: /secrets/aws-cli/.aws/config + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled .Values.volumeMounts }} + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- if .Values.docrootPersistence.enabled }} + - name: docroot + mountPath: {{ .Values.docrootPersistence.mountPath }} + {{- else if .Values.brandingNavbarLogos.enabled }} + - name: branding-writable-docroot + mountPath: /dv/docroot + {{- end }} + {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.initdFromChart.enabled }} + - name: init-d + mountPath: {{ .Values.configMap.mountPath }} + readOnly: true + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: init-d + mountPath: /opt/payara/scripts/init.d/006-s3-aws-storage.sh + subPath: 006-s3-aws-storage.sh + readOnly: true + - name: aws-cli + mountPath: /secrets/aws-cli/.aws + readOnly: true + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.persistence.enabled .Values.docrootPersistence.enabled .Values.brandingNavbarLogos.enabled .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled .Values.volumes .Values.solrInit.enabled }} + volumes: + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ if empty (.Values.persistence.existingClaim | default "") }}{{ include "dataverseup.fullname" . }}-data{{ else }}{{ .Values.persistence.existingClaim }}{{ end }} + {{- end }} + {{- if .Values.docrootPersistence.enabled }} + - name: docroot + persistentVolumeClaim: + claimName: {{ if empty (.Values.docrootPersistence.existingClaim | default "") }}{{ include "dataverseup.fullname" . }}-docroot{{ else }}{{ .Values.docrootPersistence.existingClaim }}{{ end }} + {{- end }} + {{- if .Values.brandingNavbarLogos.enabled }} + - name: branding-navbar-cm + configMap: + name: {{ include "dataverseup.fullname" . }}-branding-navbar + defaultMode: 0444 + items: + - key: logo.svg + path: logo.svg + {{- if not .Values.docrootPersistence.enabled }} + - name: branding-writable-docroot + emptyDir: {} + {{- end }} + {{- end }} + {{- if or .Values.configMap.enabled .Values.mail.enabled .Values.awsS3.enabled .Values.initdFromChart.enabled }} + - name: init-d + configMap: + name: {{ include "dataverseup.fullname" . }}-config + defaultMode: 0755 + {{- end }} + {{- if .Values.awsS3.enabled }} + - name: aws-cli + secret: + secretName: {{ required "awsS3.existingSecret is required when awsS3.enabled=true" .Values.awsS3.existingSecret | quote }} + items: + - key: {{ .Values.awsS3.secretKeys.credentials | quote }} + path: credentials + - key: {{ .Values.awsS3.secretKeys.config | quote }} + path: config + {{- end }} + {{- if .Values.solrInit.enabled }} + - name: solr-init-script + configMap: + name: {{ include "dataverseup.fullname" . }}-solr-init-script + defaultMode: 0555 + - name: solr-dataverse-conf + configMap: + name: {{ .Values.solrInit.confConfigMap | quote }} + defaultMode: 0444 + {{- end }} + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/dataverseup/templates/hpa.yaml b/charts/dataverseup/templates/hpa.yaml new file mode 100644 index 0000000..6abba32 --- /dev/null +++ b/charts/dataverseup/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "dataverseup.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/ingress.yaml b/charts/dataverseup/templates/ingress.yaml new file mode 100644 index 0000000..72d1556 --- /dev/null +++ b/charts/dataverseup/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "dataverseup.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-deployment.yaml b/charts/dataverseup/templates/internal-solr-deployment.yaml new file mode 100644 index 0000000..858936b --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-deployment.yaml @@ -0,0 +1,112 @@ +{{- if .Values.internalSolr.enabled }} +{{- if not .Values.solrInit.enabled }}{{ fail "internalSolr.enabled requires solrInit.enabled (initContainer waits for core)" }}{{- end }} +{{- if not .Values.solrInit.confConfigMap }}{{ fail "internalSolr.enabled requires solrInit.confConfigMap" }}{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "dataverseup.fullname" . }}-solr + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 8 }} + spec: + # Official solr image runs as uid 8983; PVCs often mount root-owned → solr-precreate cannot write solr.xml. + securityContext: + fsGroup: {{ .Values.internalSolr.podSecurityContext.fsGroup }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: solr-install-configset + image: busybox:1.36 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + command: + - sh + - -ec + - | + mkdir -p /configsets/dv + if [ -f /cm/solr-conf.tgz ]; then + tar -xzf /cm/solr-conf.tgz -C /configsets/dv + else + cp -a /cm/. /configsets/dv/ + fi + chown -R 8983:8983 /configsets + volumeMounts: + - name: solr-configset + mountPath: /configsets + - name: solr-conf-cm + mountPath: /cm + readOnly: true + - name: solr-datadir-perms + image: busybox:1.36 + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 0 + command: + - sh + - -ec + - | + mkdir -p /var/solr/data + chown -R {{ .Values.internalSolr.podSecurityContext.fsGroup }}:{{ .Values.internalSolr.podSecurityContext.fsGroup }} /var/solr/data + volumeMounts: + - name: solr-datadir + mountPath: /var/solr/data + containers: + - name: solr + image: {{ .Values.internalSolr.image | quote }} + imagePullPolicy: {{ .Values.internalSolr.imagePullPolicy }} + # precreate-core copies CONFIG_SOURCE as a path; "dv" alone is cwd-relative → cp: cannot stat 'dv/'. + args: ["solr-precreate", "dataverse", "/opt/solr/server/solr/configsets/dv"] + ports: + - containerPort: 8983 + name: solr-http + volumeMounts: + - name: solr-configset + mountPath: /opt/solr/server/solr/configsets + readOnly: true + - name: solr-datadir + mountPath: /var/solr/data + readinessProbe: + httpGet: + path: /solr/dataverse/admin/ping + port: 8983 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /solr/dataverse/admin/ping + port: 8983 + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 4 + {{- with .Values.internalSolr.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: solr-configset + emptyDir: {} + - name: solr-conf-cm + configMap: + name: {{ .Values.solrInit.confConfigMap | quote }} + - name: solr-datadir + {{- if .Values.internalSolr.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "dataverseup.fullname" . }}-solr-data + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-pvc.yaml b/charts/dataverseup/templates/internal-solr-pvc.yaml new file mode 100644 index 0000000..26ab34e --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-pvc.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.internalSolr.enabled .Values.internalSolr.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-solr-data + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.internalSolr.persistence.size | quote }} + {{- if .Values.internalSolr.persistence.storageClassName }} + storageClassName: {{ .Values.internalSolr.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/internal-solr-service.yaml b/charts/dataverseup/templates/internal-solr-service.yaml new file mode 100644 index 0000000..67a1a30 --- /dev/null +++ b/charts/dataverseup/templates/internal-solr-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.internalSolr.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dataverseup.fullname" . }}-solr + labels: + {{- include "dataverseup.internalSolrLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 8983 + targetPort: solr-http + protocol: TCP + name: solr-http + selector: + {{- include "dataverseup.internalSolrSelectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/dataverseup/templates/pvc-docroot.yaml b/charts/dataverseup/templates/pvc-docroot.yaml new file mode 100644 index 0000000..cc4295e --- /dev/null +++ b/charts/dataverseup/templates/pvc-docroot.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.docrootPersistence.enabled (empty (.Values.docrootPersistence.existingClaim | default "")) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-docroot + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + app.kubernetes.io/component: docroot +spec: + accessModes: + - {{ .Values.docrootPersistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.docrootPersistence.size | quote }} + {{- with .Values.docrootPersistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/pvc.yaml b/charts/dataverseup/templates/pvc.yaml new file mode 100644 index 0000000..8339099 --- /dev/null +++ b/charts/dataverseup/templates/pvc.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.persistence.enabled (empty (.Values.persistence.existingClaim | default "")) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "dataverseup.fullname" . }}-data + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} +{{- end }} diff --git a/charts/dataverseup/templates/service.yaml b/charts/dataverseup/templates/service.yaml new file mode 100644 index 0000000..f44b3cf --- /dev/null +++ b/charts/dataverseup/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "dataverseup.fullname" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "dataverseup.primarySelectorLabels" . | nindent 4 }} diff --git a/charts/dataverseup/templates/serviceaccount.yaml b/charts/dataverseup/templates/serviceaccount.yaml new file mode 100644 index 0000000..673f63b --- /dev/null +++ b/charts/dataverseup/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "dataverseup.serviceAccountName" . }} + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/dataverseup/templates/solr-init-configmap.yaml b/charts/dataverseup/templates/solr-init-configmap.yaml new file mode 100644 index 0000000..d5683b1 --- /dev/null +++ b/charts/dataverseup/templates/solr-init-configmap.yaml @@ -0,0 +1,140 @@ +{{- if .Values.solrInit.enabled }} +{{- $standalone := or .Values.internalSolr.enabled (eq .Values.solrInit.mode "standalone") }} +{{- if and (not $standalone) (not .Values.solrInit.zkConnect) }}{{ fail "solrInit.zkConnect is required when solrInit.mode is cloud and internalSolr.enabled is false" }}{{- end }} +{{- if not .Values.solrInit.confConfigMap }}{{ fail "solrInit.confConfigMap is required when solrInit.enabled (ConfigMap with Dataverse solr conf files)" }}{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dataverseup.fullname" . }}-solr-init-script + labels: + {{- include "dataverseup.labels" . | nindent 4 }} +data: + solr-init.sh: | + #!/bin/bash + set -euo pipefail + MODE="${SOLR_INIT_MODE:-cloud}" + if [[ "${MODE}" == "standalone" ]]; then + COLLECTION="${SOLR_COLLECTION:-dataverse}" + BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase or enable internalSolr for a default}" + AUTH=() + if [[ -n "${SOLR_ADMIN_USER:-}" && -n "${SOLR_ADMIN_PASSWORD:-}" ]]; then + AUTH=(-u "${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}") + fi + echo "solr-init: standalone — waiting for Solr at ${BASE} (core ${COLLECTION}) ..." + for _ in $(seq 1 120); do + if curl -sfS "${AUTH[@]}" "${BASE}/solr/${COLLECTION}/admin/ping" >/dev/null 2>&1; then + echo "solr-init: core ${COLLECTION} is reachable" + exit 0 + fi + sleep 2 + done + echo "solr-init: timeout waiting for Solr core ping" >&2 + exit 1 + fi + # SolrCloud: upload Dataverse configset to ZK, then create collection if missing. + SOLR_BIN="${SOLR_BIN:-/opt/solr/bin/solr}" + ZK="${SOLR_ZK_CONNECT:?set solrInit.zkConnect}" + CONF_DIR="${SOLR_CONF_DIR:-/solr-conf}" + COLLECTION="${SOLR_COLLECTION:-dataverse}" + CONFIGSET="${SOLR_CONFIGSET_NAME:-dataverse_config}" + BASE="${SOLR_HTTP_BASE:?set solrInit.solrHttpBase}" + SHARDS="${SOLR_NUM_SHARDS:-1}" + RF="${SOLR_REPLICATION_FACTOR:-2}" + + AUTH=() + if [[ -n "${SOLR_ADMIN_USER:-}" && -n "${SOLR_ADMIN_PASSWORD:-}" ]]; then + AUTH=(-u "${SOLR_ADMIN_USER}:${SOLR_ADMIN_PASSWORD}") + fi + + echo "solr-init: waiting for Solr admin at ${BASE} ..." + for _ in $(seq 1 90); do + if curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/info/system" >/dev/null 2>&1; then + echo "solr-init: Solr is up" + break + fi + sleep 2 + done + if ! curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/info/system" >/dev/null 2>&1; then + echo "solr-init: timeout waiting for Solr" >&2 + exit 1 + fi + + # zk upconfig -d does not reliably use the mounted ConfigMap path; stage to /tmp first. + # Do not stage under $SOLR_HOME/server/solr/configsets (not writable for the solr non-root user). + # ConfigMap ships solr-conf.tgz (kubectl --from-file=DIR omits subdirs like lang/). + STAGING="/tmp/dataverse_zk_conf" + rm -rf "${STAGING}" + mkdir -p "${STAGING}" + if [[ -f "${CONF_DIR}/solr-conf.tgz" ]]; then + echo "solr-init: extracting solr-conf.tgz into ${STAGING}" + tar --warning=no-unknown-keyword -xzf "${CONF_DIR}/solr-conf.tgz" -C "${STAGING}" || tar xzf "${CONF_DIR}/solr-conf.tgz" -C "${STAGING}" + else + echo "solr-init: no solr-conf.tgz under ${CONF_DIR} — ConfigMap is still the old flat layout (kubectl --from-file=dir omits lang/)." >&2 + echo "solr-init: listing mount (expect solr-conf.tgz or full flat conf from a ConfigMap per Dataverse Solr guide):" >&2 + ls -la "${CONF_DIR}" >&2 || true + POD_NS="" + if [[ -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then + POD_NS=$(tr -d '\n' < /var/run/secrets/kubernetes.io/serviceaccount/namespace) + echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory for your Dataverse version (see docs/DEPLOYMENT.md)." >&2 + else + echo "solr-init: fix: rebuild ConfigMap from full Solr conf directory (namespace in kubectl context)." >&2 + fi + cp -aL "${CONF_DIR}/." "${STAGING}/" + fi + UPLOAD_DIR="" + if [[ -f "${STAGING}/solrconfig.xml" ]]; then + UPLOAD_DIR="${STAGING}" + elif [[ -f "${STAGING}/conf/solrconfig.xml" ]]; then + UPLOAD_DIR="${STAGING}/conf" + else + echo "solr-init: missing solrconfig.xml — ConfigMap must be the full Dataverse Solr conf directory, not schema.xml only." >&2 + echo "solr-init: See https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr and docs/DEPLOYMENT.md" >&2 + echo "solr-init: Example: package conf dir as solr-conf.tgz ConfigMap keys (schema.xml, solrconfig.xml, lang/, …)." >&2 + echo "solr-init: Staged files under ${STAGING}:" >&2 + find "${STAGING}" -type f 2>/dev/null | sort >&2 || ls -laR "${STAGING}" >&2 || true + exit 1 + fi + + if [[ ! -f "${UPLOAD_DIR}/stopwords.txt" ]]; then + echo "solr-init: missing stopwords.txt in staged conf (ConfigMap incomplete)." >&2 + echo "solr-init: Rebuild full conf per Dataverse version; see https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr" >&2 + find "${UPLOAD_DIR}" -type f 2>/dev/null | sort | head -60 >&2 || true + exit 1 + fi + if [[ ! -f "${UPLOAD_DIR}/lang/stopwords_en.txt" ]]; then + echo "solr-init: missing lang/stopwords_en.txt — ConfigMap must include lang/ (use a packaged solr-conf.tgz or full conf dir) (not kubectl --from-file=dir)." >&2 + exit 1 + fi + + SOLR_HOME="$(cd "$(dirname "$SOLR_BIN")/.." && pwd)" + echo "solr-init: uploading configset ${CONFIGSET} to ZK ${ZK} (from ${UPLOAD_DIR})" + (cd "${SOLR_HOME}" && "$SOLR_BIN" zk upconfig -z "${ZK}" -n "${CONFIGSET}" -d "${UPLOAD_DIR}") + + echo "solr-init: checking collection ${COLLECTION}" + LIST_JSON=$(curl -sfS "${AUTH[@]}" "${BASE}/solr/admin/collections?action=LIST") + if echo "${LIST_JSON}" | grep -q "\"${COLLECTION}\""; then + echo "solr-init: collection ${COLLECTION} already exists" + exit 0 + fi + + echo "solr-init: creating collection ${COLLECTION} (shards=${SHARDS} replicas=${RF})" + CREATE_URL="${BASE}/solr/admin/collections?action=CREATE&name=${COLLECTION}&collection.configName=${CONFIGSET}&numShards=${SHARDS}&replicationFactor=${RF}&wt=json" + CREATE_BODY="${TMPDIR:-/tmp}/solr-init-create-body.json" + HTTP_CODE=$(curl -sS "${AUTH[@]}" -o "${CREATE_BODY}" -w "%{http_code}" "${CREATE_URL}") || true + if [[ "${HTTP_CODE}" != "200" ]]; then + echo "solr-init: collection CREATE failed HTTP ${HTTP_CODE}" >&2 + echo "solr-init: Solr response:" >&2 + cat "${CREATE_BODY}" >&2 2>/dev/null || true + echo "solr-init: If you see unknown configset / ZK mismatch, ensure zkConnect matches Solr (include /solr chroot if your distro uses one)." >&2 + exit 1 + fi + if command -v jq >/dev/null 2>&1; then + STATUS=$(jq -r '.responseHeader.status // empty' "${CREATE_BODY}" 2>/dev/null || true) + if [[ -n "${STATUS}" && "${STATUS}" != "0" ]]; then + echo "solr-init: collection CREATE returned non-zero status in JSON" >&2 + cat "${CREATE_BODY}" >&2 + exit 1 + fi + fi + echo "solr-init: done" +{{- end }} diff --git a/charts/dataverseup/templates/tests/test-connection.yaml b/charts/dataverseup/templates/tests/test-connection.yaml new file mode 100644 index 0000000..3567664 --- /dev/null +++ b/charts/dataverseup/templates/tests/test-connection.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "dataverseup.fullname" . }}-test-connection" + labels: + {{- include "dataverseup.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox:1.36 + command: ["wget"] + args: + - "-qO-" + - "http://{{ include "dataverseup.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}/api/info/version" + restartPolicy: Never diff --git a/charts/dataverseup/values.yaml b/charts/dataverseup/values.yaml new file mode 100644 index 0000000..a51f5ec --- /dev/null +++ b/charts/dataverseup/values.yaml @@ -0,0 +1,325 @@ +# Default values for dataverseup. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# Optional Deployment strategy (e.g. for replicaCount=1 on small clusters: maxSurge: 0, maxUnavailable: 1 +# avoids two Payara JVMs during a rolling update). +deploymentStrategy: {} + +# Container listens on 8080 (Payara). Service can expose 80 for Ingress. +container: + port: 8080 + +image: + repository: gdcc/dataverse + pullPolicy: IfNotPresent + # Docker Hub uses -noble-rN tags (see .env VERSION); there is no bare 6.10.1 tag. + # If empty, templates fall back to Chart.AppVersion. + tag: "6.10.1-noble-r0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + # targetPort is the named port on the pod (http → container.port). + +# Load env vars from existing cluster objects (place after extraEnvFrom in pod spec; explicit +# extraEnvVars below override same keys). Prefer Secrets for credentials — create them out-of-band +# (kubectl, External Secrets, Sealed Secrets), not in Helm values, so they are not stored in release history. +extraEnvFrom: [] +# - secretRef: +# name: dataverse-credentials +# - configMapRef: +# name: dataverse-nonsecret-config + +# Pass through to the Deployment; use ops/*-deploy.tmpl.yaml + envsubst for per-env values. +extraEnvVars: [] +# - name: EXAMPLE +# value: "from-template" + +livenessProbe: + httpGet: + path: /api/info/version + port: http + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 6 + +readinessProbe: + httpGet: + path: /api/info/version + port: http + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 6 + +# Optional startupProbe (omit key to disable). While failing, liveness/readiness do not run — avoids SIGTERM +# during slow Payara boot when :8080 still refuses connections. See deploy/k3d/values-k3d.yaml.in. + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# RWO PVC for Payara/Dataverse file storage (e.g. /data). Use 1 replica or a ReadWriteMany +# storage class if you scale out. Set existingClaim to bind a PVC created outside this chart. +persistence: + enabled: false + mountPath: /data + size: 20Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" + +# Optional second PVC for DATAVERSE_FILES_DOCROOT (gdcc image default: /dv/docroot). Same pattern as legacy +# dataverse-k8s "docroot" volume, but correct path for gdcc/dataverse — /logos/*, theme uploads, branding. +# https://guides.dataverse.org/en/latest/container/app-image.html#locations +# With RWO EBS, set podSecurityContext.fsGroup: 1000 so user payara can write the volume. +docrootPersistence: + enabled: false + mountPath: /dv/docroot + size: 6Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" + +# Navbar logo from chart (symlink to branding/docroot/logos/navbar/logo.svg). Uses a writable emptyDir on +# /dv/docroot (or copies onto docrootPersistence PVC) — read-only ConfigMap mounts break ConfigCheckService. +brandingNavbarLogos: + enabled: false + initImage: busybox:1.36 + docrootUid: 1000 + docrootGid: 1000 + +# Ship the repo **scripts/init.d/*.sh** bundle into the Payara ConfigMap (compose parity: ./scripts/init.d → /opt/payara/init.d). +# When true, all scripts under charts/dataverseup/files/init.d/*.sh are included; optional configMap.data overlays. +# Some scripts assume compose paths (MinIO, /opt/payara/triggers, etc.) — review before enabling in production. +initdFromChart: + enabled: false + +# Optional ConfigMap created by this chart; keys become filenames under mountPath (e.g. init scripts). +configMap: + enabled: false + mountPath: /opt/payara/init.d + data: {} + # example: + # data: + # "99-custom.sh": | + # #!/bin/sh + # echo ok + +# Mount compose-compatible mail init script (010-mailrelay-set.sh) under configMap.mountPath. +# Pod env (from extraEnvVars): system_email, mailhost, mailuser, no_reply_email, smtp_password, smtp_port, +# socket_port, smtp_auth, smtp_starttls, smtp_type, smtp_enabled — often mapped from SMTP_* in your env template. +# Payara ADMIN_USER / PASSWORD_FILE come from the gdcc image. If system_email is unset, the script no-ops. +mail: + enabled: false + +# Escape hatch: mount Secrets, extra PVCs, projected volumes, or host paths (compose-style /secrets). +# For nested paths from a Secret, use a projected volume: +# volumes: +# - name: app-secrets +# projected: +# sources: +# - secret: +# name: dataverse-secrets +# items: +# - key: db_password +# path: db/password +# - key: admin_password +# path: admin/password +# volumeMounts: +# - name: app-secrets +# mountPath: /secrets +# readOnly: true +volumes: [] + +volumeMounts: [] + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Optional raw initContainers run before the main container (same idea as Hyrax chart solrPreSetupInitContainer). +solrPreSetupInitContainer: [] + +# Dedicated Solr for this release only — not a shared cluster Solr service or SolrCloud pool. Official +# image, single core "dataverse" via solr-precreate + solrInit.confConfigMap (solr-conf.tgz or flat keys). +# Pair with solrInit.enabled and solrInit.mode: standalone (default). solrHttpBase can stay empty: the chart +# fills http://-solr..svc.cluster.local:8983 when internalSolr is enabled. +# Pin matches docker-compose.yml (IQSS conf/solr for Solr 9 / Lucene 9.x). On Apple Silicon, prefer amd64 +# nodes or cluster image policy — Compose uses platform linux/amd64 for Solr/Dataverse. +internalSolr: + enabled: false + image: solr:9.10.1 + imagePullPolicy: IfNotPresent + # Official solr image uses uid/gid 8983; fsGroup + init chown make PVC/emptyDir writable for solr-precreate. + podSecurityContext: + fsGroup: 8983 + resources: {} + persistence: + enabled: false + size: 5Gi + storageClassName: "" + +# SolrCloud bootstrap before Payara starts (same pattern as Hyrax load-solr-config initContainer: +# https://github.com/samvera/hyrax/blob/main/chart/hyrax/templates/deployment.yaml ). +# Uses a Solr image for `solr zk upconfig`, not the Dataverse image (which has no Hyrax shell scripts). +# Provide a ConfigMap whose keys are Solr conf filenames (schema.xml, solrconfig.xml, …) from the Dataverse +# release that matches your Dataverse version — see https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr +# Image aligns with internalSolr / Compose (Solr 9). standalone mode only needs curl+bash; cloud mode runs +# `solr zk upconfig` — use a Solr major that matches your SolrCloud nodes; override solrBin if you use Bitnami. +solrInit: + enabled: false + image: solr:9.10.1 + imagePullPolicy: IfNotPresent + # CLI path inside solrInit.image (official Solr 9: /opt/solr/bin/solr; Bitnami legacy: /opt/bitnami/solr/bin/solr). + solrBin: /opt/solr/bin/solr + securityContext: + runAsUser: 8983 + runAsGroup: 8983 + runAsNonRoot: true + # standalone = wait for core ping (default; use with internalSolr — new Solr pod with this deploy). + # cloud = zk upconfig + Collections API — only if you point at an existing SolrCloud + ZK (not the in-chart Solr). + mode: standalone + # ZooKeeper connect string (required only when mode: cloud), e.g. zk-0.zk-hs:2181/solr + zkConnect: "" + # Solr admin base URL (no path). Leave empty when internalSolr.enabled — chart sets the in-release Service URL. + # For a shared cluster Solr or SolrCloud front door, set explicitly, e.g. http://solr.mysolr.svc.cluster.local:8983 + solrHttpBase: "" + collection: dataverse + configSetName: dataverse_config + numShards: 1 + replicationFactor: 2 + # Name of an existing ConfigMap in the release namespace (keys = files under conf/) + confConfigMap: "" + # Optional Secret with SOLR_ADMIN_USER / SOLR_ADMIN_PASSWORD for Solr basic auth + existingSecret: "" + # If existingSecret is empty, optional inline auth (prefer Secret) + adminUser: "" + adminPassword: "" + resources: {} + +# AWS S3 file storage: Secret mount + env (aws_bucket_name, …) plus bundled 006-s3-aws-storage.sh mounted at +# /opt/payara/scripts/init.d/ (base image entrypoint runs that path before Payara starts). See docs/DEPLOYMENT.md. +awsS3: + enabled: false + # Name of a Secret you create out-of-band (keys = secretKeys below). Required when enabled. + existingSecret: "" + bucketName: "your-bucket-name" + # Leave empty for real Amazon S3 (recommended). Set for MinIO / S3-compatible only (e.g. https://minio:9000). + endpointUrl: "" + # AWS SigV4 signing region (e.g. us-west-2). Must match bucket; set in mounted config + AWS_REGION on Deployment. + region: "us-west-2" + # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. + profile: default + secretKeys: + credentials: credentials + config: config + +# gdcc/configbaker Job (post-install hook by default). +# - mode: oneShot — `bootstrapJob.command` only (e.g. bootstrap.sh dev), same as a minimal container demo. +# - mode: compose — like docker compose: configbaker writes API token, then apply-branding.sh, then seed-content.sh +# (repo scripts + fixtures baked into a ConfigMap). Uses emptyDir for bootstrap.env inside the Job only. +# - helmHook=true: runs once on install only (not upgrade). helmHook=false: plain Job (immutable spec on upgrade). +bootstrapJob: + enabled: false + mode: oneShot + helmHook: true + image: + repository: gdcc/configbaker + tag: "" + pullPolicy: IfNotPresent + command: + - bootstrap.sh + - dev + # Empty = http://..svc.cluster.local: (matches DATAVERSE_URL on Deployment). + dataverseUrl: "" + timeout: 20m + backoffLimit: 2 + ttlSecondsAfterFinished: 86400 + resources: {} + compose: + # Wait for /api/info/version before bootstrap.sh (Payara can be slow after install). + waitMaxSeconds: 900 + waitSleepSeconds: 5 + # Run fixtures/seed after branding (set false for bootstrap+branding only). + seed: true + # When the DB is already bootstrapped, configbaker prints "skipping" and may not write API_TOKEN to the file. + # Optional admin API token Secret (same name can stay in values for the whole lifecycle). The Job uses + # secretKeyRef optional:true so a missing Secret on first install does not block the pod — configbaker + # still writes API_TOKEN. After bootstrap, create the Secret for re-runs / already-bootstrapped DB: + # kubectl create secret generic dataverse-admin-api-token -n YOUR_NS --from-literal=token=UUID + existingAdminApiTokenSecret: "" + adminApiTokenSecretKey: token + # post-upgrade hook: branding + seed only (no configbaker). Use when the release was upgraded and the + # post-install hook never ran compose, or you need to re-apply after changing branding/fixtures in the chart. + # Create a Secret first, e.g. kubectl create secret generic dv-admin-api-token -n ns --from-literal=token=UUID + postUpgradeBrandingSeedJob: + enabled: false + existingSecret: "" + secretKey: token diff --git a/docker-compose.yml b/docker-compose.yml index db08f14..edabcbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: - "POSTGRES_PORT" volumes: - postgres_data:/var/lib/postgresql/data - - ./triggers:/triggers + - ./scripts/triggers:/triggers solr: <<: *env-from-file @@ -184,8 +184,8 @@ services: # /dv/docroot/logos//... (theme uploads) stays on the container filesystem. - ./branding/docroot/logos/navbar:/dv/docroot/logos/navbar:ro - ./branding/docroot/branding:/dv/docroot/branding:ro - - ./init.d:/opt/payara/init.d - - ./triggers:/opt/payara/triggers + - ./scripts/init.d:/opt/payara/init.d + - ./scripts/triggers:/opt/payara/triggers - ./config/schema.xml:/opt/payara/dvinstall/schema.xml labels: - "traefik.enable=true" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..0d7115e --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,280 @@ +# Deployment (working document) + +Rough notes for standing up Dataverse for Notch8: **Docker Compose** (lab), **Kubernetes / Helm**, optional **GitHub Actions** deploys, and a shared **learnings** table. Extend into a full runbook as you validate each environment. + +**Reference:** Source — [github.com/notch8/dataverseup](https://github.com/notch8/dataverseup). An **active** deployment from this approach is at [demo-dataverseup.notch8.cloud](https://demo-dataverseup.notch8.cloud/) (seeded demo content for smoke tests). + +## Ticket context (internal) + +- **Target:** Dataverse **v6.10** on **AWS** by **April 7, 2026** — functional demo, not necessarily production-hardened. +- **Deliverable:** Working deployment **and documented process + learnings** (this document). + +## Docker Compose (local / lab) + +See repository **[README.md](../README.md)** — `docker compose up` after `.env` and `secrets/` from examples. + +--- + +## Kubernetes: Helm chart + +The following sections describe how to install the **`dataverseup`** Helm chart from this repository. + +**Prerequisites:** Helm 3, a Kubernetes cluster, `kubectl` configured, a **PostgreSQL** database reachable from the cluster (in-cluster or managed), and a **StorageClass** for any PVCs you enable. + +**Chart path:** `charts/dataverseup` + +### `bin/helm_deploy` (recommended wrapper) + +From the **repository root**, installs or upgrades the chart with **`--install`**, **`--atomic`**, **`--create-namespace`**, and a default **`--timeout 30m0s`** (Payara first boot is slow). + +```text +./bin/helm_deploy RELEASE_NAME NAMESPACE +``` + +Pass extra Helm flags with **`HELM_EXTRA_ARGS`** (values file, longer timeout, etc.). If you pass a **second `--timeout`** in `HELM_EXTRA_ARGS`, it overrides the default (Helm uses the last value). + +```bash +HELM_EXTRA_ARGS="--values ./your-values.yaml --wait --timeout 45m0s" ./bin/helm_deploy my-release my-namespace +``` + +### What the chart deploys + +- **Dataverse** (`gdcc/dataverse`) — Payara on port **8080**; Service may expose **80** → target **8080** for Ingress compatibility. +- **Optional bootstrap Job** (`gdcc/configbaker`) — usually a **Helm post-install hook** (`bootstrapJob.helmHook: true`). **`bootstrapJob.mode: oneShot`** runs **`bootstrapJob.command`** only (default: `bootstrap.sh dev` — FAKE DOI, `dataverseAdmin`, etc.). **`bootstrapJob.mode: compose`** mirrors local Docker Compose: wait for the API, run configbaker with a writable token file on `emptyDir`, then **`apply-branding.sh`** and **`seed-content.sh`** (fixtures baked into a ConfigMap). Tune waits with **`bootstrapJob.compose`** and allow a longer **`bootstrapJob.timeout`** when seeding. +- **Optional dedicated Solr** (`internalSolr`) — a **new** Solr Deployment/Service in the **same release and namespace** as Dataverse (not wiring into someone else’s shared “cluster Solr”). Default **`solrInit.mode`** is **`standalone`**: the Dataverse pod waits for that Solr core before starting. Use **`solrInit.mode: cloud`** only when Dataverse talks to **SolrCloud + ZooKeeper** you operate separately. +- **Optional S3** — `awsS3.enabled` mounts AWS credentials and ships the S3 init script. + +#### Branding (navbar logo + Admin API settings) + +1. **Navbar SVG** — Enable **`brandingNavbarLogos.enabled`** so an init container copies **`branding/docroot/logos/navbar/logo.svg`** from the chart onto **`/dv/docroot/logos/navbar/logo.svg`** (needs **`docrootPersistence`** or the chart’s emptyDir docroot fallback). Match **`LOGO_CUSTOMIZATION_FILE`** in **`branding/branding.env`** to the web path (e.g. `/logos/navbar/logo.svg`). + +2. **Admin settings** (installation name, footer, optional custom header/footer CSS paths) — Edit **`branding/branding.env`** in the repo. The chart embeds it in the **`…-bootstrap-chain`** ConfigMap when **`bootstrapJob.mode: compose`**. The post-install Job runs **`apply-branding.sh`**, which PUTs those settings via the Dataverse Admin API using the admin token from configbaker. + +3. **Custom HTML/CSS files** — Add them under **`branding/docroot/branding/`** in the repo, set **`HEADER_CUSTOMIZATION_FILE`**, etc. in **`branding.env`** to **`/dv/docroot/branding/...`**, and ship those files into the pod (extra **`volumeMounts`** / **`configMap`** or bake into an image). The stock chart does not mount the whole **`branding/docroot/branding/`** tree on the main Deployment; compose only ships **`branding.env`** and the logo via **`brandingNavbarLogos`**. + +4. **After `helm upgrade`** — The post-install hook does **not** re-run. To re-apply branding, use **`bootstrapJob.compose.postUpgradeBrandingSeedJob`** with a Secret holding **`DATAVERSE_API_TOKEN`**, or run **`scripts/apply-branding.sh`** locally/cron with **`DATAVERSE_INTERNAL_URL`** and a token. + +The chart does **not** install PostgreSQL by default. Supply DB settings with **`extraEnvVars`** and/or **`extraEnvFrom`** (recommended: Kubernetes **Secret** for passwords). + +#### Recommended Solr layout: new instance with this deploy + +Enable **`internalSolr.enabled`**, **`solrInit.enabled`**, keep **`solrInit.mode: standalone`**, and supply **`solrInit.confConfigMap`**. Leave **`solrInit.solrHttpBase` empty** — the chart sets the Solr admin URL to the in-release Service (`http://-solr..svc.cluster.local:8983`). Point your app Secret at that same host/port and core (see table below). You do **not** need an existing Solr installation in the cluster. + +### Docker Compose vs Helm (Solr) + +Local **`docker-compose.yml`** and this chart both target **official Solr 9** (`solr:9.10.1`) and IQSS **`conf/solr`** files vendored under repo **`config/`** (refresh from IQSS `develop` or a release tag as in the root **`README.md`**). + +| | Docker Compose | Helm (`internalSolr` + `solrInit`) | +|---|----------------|-----------------------------------| +| Solr image pin | `solr:9.10.1` | `internalSolr.image` / `solrInit.image` default `solr:9.10.1` | +| Default core name | **`collection1`** (see `scripts/solr-initdb/01-ensure-core.sh`) | **`dataverse`** (`solr-precreate` in `internal-solr-deployment.yaml`) | +| App Solr address | `SOLR_LOCATION=solr:8983` (host:port) | With **`internalSolr.enabled`**, the chart sets **`DATAVERSE_SOLR_HOST`**, **`DATAVERSE_SOLR_PORT`**, **`DATAVERSE_SOLR_CORE`**, **`SOLR_SERVICE_*`**, and **`SOLR_LOCATION`** to the in-release Solr Service and **`solrInit.collection`** (default **`dataverse`**). The GDCC `ct` profile otherwise defaults to host **`solr`** and core **`collection1`**, which breaks Kubernetes installs if unset. | + +Compose only copies **`schema.xml`** and **`solrconfig.xml`** into the core after precreate. **SolrCloud** (`solrInit.mode: cloud`) still needs a **full** conf tree or **`solr-conf.tgz`** (including `lang/`, `stopwords.txt`, etc.) for `solr zk upconfig` — see [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +#### `solrInit` image: standalone (default) vs SolrCloud + +- **Standalone** (default, with **`internalSolr`**): the initContainer **waits** for `/solr//admin/ping` via `curl`; the default **`solr:9.10.1`** image is sufficient. This matches launching a **solo Solr** with the chart instead of consuming a shared cluster Solr Service. +- **Cloud / ZooKeeper** (optional): set **`solrInit.mode: cloud`** and **`solrInit.zkConnect`** when Dataverse uses **SolrCloud** you run elsewhere. The same container runs **`solr zk upconfig`**; use a Solr **major** compatible with that cluster. Override **`solrInit.image`**, **`solrInit.solrBin`**, and **`solrInit.securityContext`** if you use a vendor image (e.g. legacy Bitnami). + +### Install flow (recommended order) + +1. **Create namespace** + `kubectl create namespace ` + +2. **Database** + Provision Postgres and a database/user for Dataverse. Note the service DNS name inside the cluster (e.g. `postgres..svc.cluster.local`). + +3. **Solr configuration ConfigMap** (if using `solrInit` / `internalSolr`) + Dataverse needs a **full** Solr configuration directory for its version — not `schema.xml` alone. Build a ConfigMap whose keys are the files under that conf directory (or a single `solr-conf.tgz` as produced by your packaging process). See [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr). + +4. **Application Secret** (example name `dataverse-app-env`) + Prefer `stringData` for passwords. Include at least the variables the GDCC image expects for JDBC and Solr (mirror what you use in Docker Compose `.env`). Typical keys include: + + - `DATAVERSE_DB_HOST`, `DATAVERSE_DB_USER`, `DATAVERSE_DB_PASSWORD`, `DATAVERSE_DB_NAME` + - `POSTGRES_SERVER`, `POSTGRES_PORT`, `POSTGRES_DATABASE`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `PGPASSWORD` + - Solr: `SOLR_LOCATION` or `DATAVERSE_SOLR_HOST` / `DATAVERSE_SOLR_PORT` / `DATAVERSE_SOLR_CORE` (match your Solr deployment) + - Public URL / hostname: `DATAVERSE_URL`, `hostname`, `DATAVERSE_SERVICE_HOST` (used by init scripts and UI) + - Optional: `DATAVERSE_PID_*` for FAKE DOI (see default chart comments and [container demo docs](https://guides.dataverse.org/en/latest/container/running/demo.html)) + +5. **Values file** + Start from `charts/dataverseup/values.yaml` and override with a small values file of your own. At minimum for a first install: + + - `persistence.enabled: true` (file store) + - `extraEnvFrom` pointing at your Secret + - If using dedicated in-chart Solr: `internalSolr.enabled`, `solrInit.enabled`, `solrInit.confConfigMap`, `solrInit.mode: standalone` (default). Omit `solrInit.solrHttpBase` to use the auto-derived in-release Solr Service URL + - `bootstrapJob.enabled: true` for first-time seeding + +6. **Lint and render** + + ```bash + helm lint charts/dataverseup -f your-values.yaml + helm template dataverseup charts/dataverseup -f your-values.yaml > /tmp/manifests.yaml + ``` + +7. **Install** + + Using the wrapper (from repo root): + + ```bash + HELM_EXTRA_ARGS="--values ./your-values.yaml --wait" ./bin/helm_deploy + ``` + + Raw Helm (equivalent shape): + + ```bash + helm upgrade --install charts/dataverseup -n -f your-values.yaml --wait --timeout 45m + ``` + +8. **Smoke tests** + + - `kubectl get pods -n ` + - Bootstrap job logs (if enabled): `kubectl logs -n job/...-bootstrap` + - API: port-forward or Ingress → `GET /api/info/version` should return **200** + - UI login (default bootstrap admin from configbaker **dev** profile — **change** before any shared environment) + +9. **Helm test** (optional) + + ```bash + helm test -n + ``` + +### GitHub Actions — Deploy workflow + +The **[.github/workflows/deploy.yaml](../.github/workflows/deploy.yaml)** job uses the GitHub **Environment** named by the `environment` workflow input (e.g. `demo`). It must match **`ops/-deploy.tmpl.yaml`**. The **Prepare kubeconfig and render deploy values** step runs **`envsubst` only for selected secrets** (`DB_PASSWORD`, `SMTP_PASSWORD`) and **`GITHUB_RUN_ID`**. **Public URLs, ingress, in-cluster Solr/Dataverse Service DNS, `:SystemEmail` / JavaMail From addresses (demo: both `support@notch8.com`), S3 bucket name, and Postgres identifiers are plain literals** in that file — edit them there when the environment changes (they must match your Helm release/namespace, e.g. `demo-dataverseup`). + +**Secrets (typical, per Environment):** `DB_PASSWORD`, `KUBECONFIG_FILE` (base64), optional mail: **`SMTP_PASSWORD`** (SendGrid API key for demo). Demo values fix **`system_email`** and **`no_reply_email`** to **`support@notch8.com`** in **`ops/demo-deploy.tmpl.yaml`** (no `SYSTEM_EMAIL` / `NO_REPLY_EMAIL` secrets required). + +**Repository or Environment variables (optional):** + +| Variable | Purpose | Default if unset | +|----------|---------|------------------| +| `DEPLOY_TOOLBOX_IMAGE` | Job `container` image | `dtzar/helm-kubectl:3.9.4` | +| `HELM_CHART_PATH` | Path passed to `helm` / `bin/helm_deploy` | `./charts/dataverseup` | +| `HELM_APP_NAME` | `app.kubernetes.io/name` for `kubectl rollout status` | `github.event.repository.name` | +| `DEPLOY_ROLLOUT_TIMEOUT` | Rollout wait | `10m` | +| `DEPLOY_BOOTSTRAP_JOB_TIMEOUT` | Bootstrap Job wait | `25m` | + +Default Helm **release** and **namespace** are **`-`** (e.g. `demo-dataverseup`). Override with workflow inputs `k8s_release_name` / `k8s_namespace` when needed. + +For **demo**, SMTP host, ports, auth flags, and **`support@notch8.com`** addresses live only in **`ops/demo-deploy.tmpl.yaml`** — the workflow does **not** pass GitHub Variables for those; change the template (or add `${VAR}` placeholders and extend `envsubst`) if you need per-environment overrides. + +**Migrating or renaming a release:** Update the literals in **`ops/-deploy.tmpl.yaml`** (ingress hosts, `dataverse_*` / `DATAVERSE_*` / `hostname`, `solrHttpBase`, `SOLR_*`, `DATAVERSE_URL`, `awsS3.bucketName`, DB names, etc.) so they match the new Helm release and namespace; then align Postgres, S3, TLS, and running workloads. + +#### SMTP (JavaMail) — enable, secrets, and how to test + +1. **Chart:** set **`mail.enabled: true`** so **`010-mailrelay-set.sh`** is in the ConfigMap. Pod env must include the names the script reads: **`system_email`**, **`mailhost`**, **`mailuser`**, **`no_reply_email`**, **`smtp_password`**, **`smtp_port`**, **`socket_port`**, **`smtp_auth`**, **`smtp_starttls`**, **`smtp_type`**, **`smtp_enabled`** (see **`ops/demo-deploy.tmpl.yaml`** and **`scripts/init.d/010-mailrelay-set.sh`**). **`smtp_enabled`** must not be `false` / `0` / `no`, and **`system_email`** must be non-empty or the script no-ops. + +2. **GitHub Environment (demo):** Add **Secret** **`SMTP_PASSWORD`** (SendGrid API key). **`ops/demo-deploy.tmpl.yaml`** sets **`DATAVERSE_MAIL_SYSTEM_EMAIL`** / **`DATAVERSE_MAIL_SUPPORT_EMAIL`** to **`support@notch8.com`** and **`DATAVERSE_MAIL_MTA_*`** for SendGrid (587 + STARTTLS + **`apikey`** user). Since **Dataverse 6.2+**, outbound mail uses these **MicroProfile** settings ([SMTP/Email in the Dataverse Guide](https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration)); **`010-mailrelay-set.sh`** (Payara JavaMail + `:SystemEmail`) is **not sufficient alone**. Set **`DATAVERSE_MAIL_DEBUG`** to **`true`** in the template temporarily for verbose mail logs ([`dataverse.mail.debug`](https://guides.dataverse.org/en/latest/installation/config.html)). **Restart pods** after changing mail env (MTA session is cached). + +3. **Verify config:** After the pod is ready, check the setting: `curl -sS "https:///api/admin/settings/:SystemEmail"` (use a superuser API token if the endpoint requires auth on your version). In the UI, use **Contact** / support mail, **Forgot password**, or user signup (if enabled) and confirm delivery (and provider dashboard / spam folder). + +4. **Logs:** `kubectl logs -n deploy/-dataverseup` (or your deployment name) and look for **`010-mailrelay`** / **`asadmin`** errors during startup. + +5. **Docker Compose (local):** Set the same variable names in **`.env`** (see **`.env.example`** SMTP block). They are passed through on the **dataverse** service. Restart **dataverse** and tail **`docker compose logs -f dataverse`** during Payara init. + +6. **If the UI says the message was sent but nothing arrives:** + - **`010-mailrelay-set.sh` runs only when Payara starts.** Changing `SMTP_PASSWORD` in GitHub without **rolling/restarting** Dataverse pods leaves the old JavaMail session (or none). After fixing secrets, **redeploy or delete pods** so init runs again. + - **Confirm rendered values:** open the generated `ops/-deploy.yaml` (CI artifact or run `envsubst` locally) and check `smtp_password` is non-empty (not the literal `${SMTP_PASSWORD}`) and `system_email` / `no_reply_email` are **`support@notch8.com`** for demo. + - **SendGrid:** use the **full API key** as `SMTP_PASSWORD`, **`mailuser` = `apikey`**, port **587**, **`smtp_auth` / `smtp_type`** as in **`ops/demo-deploy.tmpl.yaml`**. In [SendGrid Activity](https://app.sendgrid.com/email_activity), see bounces, blocks, and “unauthenticated” sends. **Verify** the **From** address (`no_reply_email`) via Single Sender or domain authentication. + - **Payara:** successful sends often log **nothing** at default levels. Exec into the Dataverse container and run: + `asadmin --user "$ADMIN_USER" --passwordfile "$PASSWORD_FILE" list-javamail-resources` + You should see **`mail/notifyMailSession`**. If it is missing, init did not configure mail (wrong env, `smtp_enabled`, or script failed). + **Domain logs (GDCC base image):** Payara lives under **`/opt/payara/appserver`**, not `/opt/payara/glassfish` alone. Typical file: + `/opt/payara/appserver/glassfish/domains/domain1/logs/server.log` + Or discover: `find /opt/payara -name server.log 2>/dev/null`. Much application output also goes to **`kubectl logs`** on the main container (`dataverseup`). + - **Network:** from the pod, `nc -zv smtp.sendgrid.net 587` (or your `mailhost`) must succeed if the cluster egress allows it. + +7. **Why you don’t see Rails-style “Sent mail” lines:** **ActionMailer** logs each delivery at INFO by default. **Dataverse 6.2+** uses **`dataverse.mail.*`** / **`DATAVERSE_MAIL_*`**; with **`DATAVERSE_MAIL_DEBUG=true`** you get the **supported** verbose logging described in the [installation guide](https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration) (set in **`ops/demo-deploy.tmpl.yaml`**, then roll pods — **disable after debugging**). + + **Lower-level JavaMail trace (optional):** `-Dmail.debug=true` on **`JVM_OPTS`** still works for raw SMTP wire logs but can leak credentials; prefer **`DATAVERSE_MAIL_DEBUG`** first. + + **Closest thing to a delivery log:** **[SendGrid Activity](https://app.sendgrid.com/email_activity)** (accept / bounce / block). + +### Ingress and TLS + +Set `ingress.enabled: true`, `ingress.className` to your controller (e.g. `nginx`, `traefik`), and hosts/TLS to match your DNS. Payara serves **HTTP** on 8080; the Service fronts it on port **80** so Ingress backends stay HTTP. + +If you terminate TLS or expose the app on a **non-default host port**, keep **`DATAVERSE_URL`** and related hostname settings aligned with the URL users and the app use. + +### Payara init scripts (DRY with Compose) + +The chart embeds the S3 and mail relay scripts from **`scripts/init.d/`** via symlinks under `charts/dataverseup/files/init.d/`. Edit **`scripts/init.d/006-s3-aws-storage.sh`** or **`scripts/init.d/010-mailrelay-set.sh`** once; both Compose mounts and Helm ConfigMaps stay aligned. `helm package` resolves symlink content into the tarball. + +Set **`initdFromChart.enabled: true`** in values to include **all** `files/init.d/*.sh` in the same ConfigMap (compose parity with mounting **`./scripts/init.d`**). Keep **`INIT_SCRIPTS_FOLDER`** (or the image default) pointed at **`/opt/payara/init.d`**. Review MinIO- and webhook/trigger scripts (repo **`scripts/triggers/`**, Compose → `/opt/payara/triggers`) before enabling in a cluster that does not mount those paths. + +### S3 file storage + +1. Set `awsS3.enabled: true`, `awsS3.existingSecret`, `bucketName`, `region`, and `profile` in values. The IAM principal behind the Secret needs S3 access to that bucket. For **Amazon S3**, leave **`endpointUrl` empty** so Payara’s init script does not set `custom-endpoint-url` (a regional `https://s3….amazonaws.com` URL there commonly causes upload failures). Set **`endpointUrl` only** for MinIO or other S3-compatible endpoints. + +2. Create a **generic** Secret in the **same namespace** as the Helm release, **before** pods that mount it start. Key names must match `awsS3.secretKeys` (defaults below): the values are the **raw file contents** of `~/.aws/credentials` and `~/.aws/config`. + + - `credentials` — ini format; the profile block header (e.g. `[default]` or `[my-profile]`) must match **`awsS3.profile`**. + - `config` — ini format; for `profile: default` use `[default]` with `region = ...`. For a named profile use `[profile my-profile]` and the same region as `awsS3.region` unless you know you need otherwise. + +3. **Examples** (replace `NAMESPACE`, keys, region, and secret name if you changed `existingSecret`): + + ```sh + NS=NAMESPACE + kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - + + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-file=credentials="$HOME/.aws/credentials" \ + --from-file=config="$HOME/.aws/config" \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + Inline `[default]` user (no local files): + + ```sh + kubectl -n "$NS" create secret generic aws-s3-credentials \ + --from-literal=credentials="[default] + aws_access_key_id = AKIA... + aws_secret_access_key = ... + " \ + --from-literal=config="[default] + region = us-west-2 + " \ + --dry-run=client -o yaml | kubectl apply -f - + ``` + + If you use **temporary credentials** (assumed role / STS), add a line to the credentials profile: `aws_session_token = ...`. Rotate before expiry or automate renewal. + +4. After creating or updating the Secret, **restart** the Dataverse Deployment (or delete its pods) so the volume is remounted. The chart sets `AWS_SHARED_CREDENTIALS_FILE` and `AWS_CONFIG_FILE` to the mounted paths. + +**Note:** The Java AWS SDK inside the app may not perform the same **assume-role chaining** as the AWS CLI from a complex `config` file. Prefer putting **direct** user keys or **already-assumed** temporary keys in the Secret for the app, or use EKS **IRSA** (service account + role) instead of long-lived keys if your platform supports it. + +#### Troubleshooting: `Failed to save the content of the uploaded file` + +The Native API returns **HTTP 400** with that message when the request reached Dataverse but **writing to the configured store failed**. This is not a bug in the seed script’s `jsonData` shape. + +With **`dataverse.files.storage-driver-id=S3`** (see `scripts/init.d/006-s3-aws-storage.sh`): + +1. **IAM** — The principal in your `aws-s3-credentials` Secret needs at least **`s3:PutObject`**, **`s3:GetObject`**, **`s3:DeleteObject`**, and **`s3:ListBucket`** on the target bucket (and prefixes Dataverse uses). Missing `PutObject` often surfaces exactly as this generic message; the real error is in server logs. +2. **Bucket and region** — `awsS3.bucketName` and `awsS3.region` must match the bucket. For **Amazon S3**, keep **`awsS3.endpointUrl` empty**; do not point it at `https://s3..amazonaws.com` unless you are on a non-AWS S3-compatible store. +3. **Credentials mounted** — After creating or rotating the Secret, **restart** Dataverse pods so `AWS_SHARED_CREDENTIALS_FILE` / `AWS_CONFIG_FILE` point at the new files. +4. **Logs** — Check the Dataverse Deployment pod logs for nested exceptions (`AccessDenied`, `NoSuchBucket`, SSL, etc.). + +### Upgrades + +- Bump `image.tag` / `Chart.appVersion` together with [Dataverse release notes](https://github.com/IQSS/dataverse/releases). +- Reconcile Solr conf ConfigMap when Solr schema changes. +- When upgrading **internal Solr** across a **major Solr version** (e.g. 8 → 9), use a **fresh** Solr data volume (new PVC or wipe `internalSolr` persistence) so cores are recreated; same idea as Compose (see root **`README.md`**). +- After bumping **`solrInit`** / **`internalSolr`** images, re-test **SolrCloud** installs (`solr zk` + collection create) in a non-production cluster if you use `solrInit.mode: cloud`. +- If `bootstrapJob.helmHook` is **true**, the bootstrap Job runs on **post-install only**, not on every upgrade (by design). + +--- + +## Learnings log + +Append rows as you go (Compose, cluster, CI, etc.): + +| Date | Environment | Note | +|------|-------------|------| +| | | | + +## References + +- [Running Dataverse in Docker](https://guides.dataverse.org/en/latest/container/running/index.html) (conceptual parity with container env) +- [Application image](https://guides.dataverse.org/en/latest/container/app-image.html) +- [Solr prerequisites](https://guides.dataverse.org/en/latest/installation/prerequisites.html#solr) diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml new file mode 100644 index 0000000..75d5fa1 --- /dev/null +++ b/ops/demo-deploy.tmpl.yaml @@ -0,0 +1,329 @@ +# Helm values for the "demo" GitHub Environment (staging / shared demo). +# +# **Edit literals in this file** for your environment: public hostname (ingress + dataverse_siteUrl), in-cluster +# Solr/Dataverse Service DNS (solrHttpBase + SOLR_* / DATAVERSE_URL), S3 bucket, Postgres identifiers, etc. +# They must stay consistent with your Helm **release name** and **namespace** (e.g. both `demo-dataverseup`). +# +# **envsubst** (CI: see `.github/workflows/deploy.yaml`) only injects secrets and the rollout nonce — not URLs: +# $DB_PASSWORD, $SMTP_PASSWORD, ${GITHUB_RUN_ID} +# **system_email** / **no_reply_email** are literals here (both support@notch8.com for SendGrid From + :SystemEmail). +# Local render: +# export DB_PASSWORD=... GITHUB_RUN_ID=manual-1 SMTP_PASSWORD=... +# envsubst '$GITHUB_RUN_ID $DB_PASSWORD $SMTP_PASSWORD' < ops/demo-deploy.tmpl.yaml > ops/demo-deploy.yaml +# +# Before first deploy: ConfigMap **dataverse-solr-conf** must ship Solr 9 conf (full tree or solr-conf.tgz), +# same lineage as docker-compose (IQSS conf under repo **config/** — see **docs/DEPLOYMENT.md**). +# +# With **awsS3.enabled**, create a **generic** Secret in the release namespace **before** Helm installs pods that +# mount it. The chart mounts it as AWS CLI files under `/secrets/aws-cli/.aws/`. +# +# • Secret **metadata.name** must match **awsS3.existingSecret** below (default: aws-s3-credentials). +# • Secret **data keys** must match **awsS3.secretKeys** (default: `credentials` and `config`) — file bodies in +# the same format as `~/.aws/credentials` and `~/.aws/config`. +# • **awsS3.profile** must match a profile stanza in those files: `[default]` in credentials, and either +# `[default]` or `[profile yourname]` in config (see AWS CLI docs for named profiles). +# +# From your laptop (kubeconfig aimed at the cluster), examples — set NS to your namespace (e.g. demo-dataverseup): +# +# NS=demo-dataverseup +# kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +# +# # A) Reuse local CLI files (simplest if profiles already match awsS3.profile): +# kubectl -n "$NS" create secret generic aws-s3-credentials \ +# --from-file=credentials="$HOME/.aws/credentials" \ +# --from-file=config="$HOME/.aws/config" \ +# --dry-run=client -o yaml | kubectl apply -f - +# +# # B) Two small files (avoids multiline --from-literal quoting); region must match awsS3.region: +# printf '%s\n' '[default]' 'aws_access_key_id = AKIA...' 'aws_secret_access_key = ...' > /tmp/aws-credentials +# printf '%s\n' '[default]' 'region = us-west-2' > /tmp/aws-config +# kubectl -n "$NS" create secret generic aws-s3-credentials \ +# --from-file=credentials=/tmp/aws-credentials --from-file=config=/tmp/aws-config \ +# --dry-run=client -o yaml | kubectl apply -f && rm -f /tmp/aws-credentials /tmp/aws-config +# +# # C) STS / assumed-role: add `aws_session_token = ...` under the same profile in the credentials file body. +# +# After creating or rotating the Secret, restart the Dataverse Deployment so the volume remounts. +# Full detail: **docs/DEPLOYMENT.md** (S3 file storage). + +awsS3: + enabled: true + # Kubernetes Secret you create out-of-band (see comments above). + existingSecret: "aws-s3-credentials" + bucketName: "demo-dataverseup" + # Empty for Amazon S3 — do not set a regional s3.*.amazonaws.com URL here; it breaks SDK uploads. Use endpointUrl only for MinIO / compatible stores. + endpointUrl: "" + region: us-west-2 + # Must match the profile section in mounted .aws/credentials (e.g. [default]); override in values if needed. + profile: default + secretKeys: + credentials: credentials + config: config + +replicaCount: 1 + +# CI sets GITHUB_RUN_ID (see .github/workflows/deploy.yaml) so each deploy changes this annotation and forces a +# new ReplicaSet. Local renders: export GITHUB_RUN_ID=manual-1 (or any string) when you need the same without CI. +podAnnotations: + deploy.github.com/run-id: "${GITHUB_RUN_ID}" + +# Docker Hub tag (not bare 6.10.1). Keeps deploy values explicit so Helm release history cannot stick to a bad tag. +image: + repository: gdcc/dataverse + tag: "6.10.1-noble-r0" + +resources: + limits: + memory: 4Gi + cpu: "1000m" + requests: + memory: 2Gi + cpu: 100m + +# Payara runs as uid 1000; fsGroup lets RWO volumes stay writable for /data and /dv/docroot. +podSecurityContext: + fsGroup: 1000 + +# RWO PVC for Dataverse /data — correct for replicaCount: 1. GDCC K8s guide: use ReadWriteMany for /data (and +# docroot) only for multi-instance Payara; with S3 (besties awsS3) local /data is partly temp/ingest. See +# https://k8s-docs.gdcc.io/en/latest/day1/storage.html#application-server +persistence: + enabled: true + mountPath: /data + size: 20Gi + accessMode: ReadWriteOnce + storageClassName: "" + +# Branding / docroot (legacy dataverse-k8s pattern: separate "docroot" PVC). gdcc/dataverse serves /logos/* from +# DATAVERSE_FILES_DOCROOT → mount at /dv/docroot. Chart creates -docroot or set existingClaim. +docrootPersistence: + enabled: true + mountPath: /dv/docroot + size: 6Gi + accessMode: ReadWriteOnce + storageClassName: "" + existingClaim: "" + +# Navbar logo file on disk (chart embeds **branding/docroot/logos/navbar/logo.svg**). Init copies it to +# `/dv/docroot/logos/navbar/logo.svg` so it matches **LOGO_CUSTOMIZATION_FILE** in **branding/branding.env** +# (API path `/logos/navbar/logo.svg`). For custom header/footer/CSS, add files under **branding/docroot/branding/** +# in the repo, extend **branding.env** with `/dv/docroot/branding/...` paths, then rely on **bootstrapJob.mode: compose** +# (below) to PUT those settings after configbaker — or run **scripts/apply-branding.sh** manually with an admin token. +brandingNavbarLogos: + enabled: true + +ingress: + enabled: true + className: nginx-ingress + annotations: + nginx.org/client-max-body-size: "0" + cert-manager.io/cluster-issuer: letsencrypt-production-dns + hosts: + - host: "demo-dataverseup.notch8.cloud" + paths: + - path: / + pathType: Prefix + - host: "*.demo-dataverseup.notch8.cloud" + paths: + - path: / + pathType: Prefix + tls: + - hosts: + - "demo-dataverseup.notch8.cloud" + - "*.demo-dataverseup.notch8.cloud" + secretName: "demo-dataverseup-tls" + +# Standalone Solr — **solr:9.10.1** matches **docker-compose.yml** and **charts/dataverseup/values.yaml** (IQSS Solr 9 conf). +internalSolr: + enabled: true + image: solr:9.10.1 + imagePullPolicy: IfNotPresent + podSecurityContext: + fsGroup: 8983 + # RWO matches GDCC: standalone Solr index — "ReadWriteOnce ... sufficient" (not SolrCloud yet). + # https://k8s-docs.gdcc.io/en/latest/day1/storage.html#index-server + persistence: + enabled: true + size: 5Gi + storageClassName: "" + resources: {} + +# Wait for in-cluster Solr core ping before Payara (standalone). For external SolrCloud, set internalSolr.enabled: false, +# mode: cloud, zkConnect, solrHttpBase, and replicationFactor — see charts/dataverseup/values.yaml. +solrInit: + enabled: true + mode: standalone + image: solr:9.10.1 + imagePullPolicy: IfNotPresent + solrBin: /opt/solr/bin/solr + securityContext: + runAsUser: 8983 + runAsGroup: 8983 + runAsNonRoot: true + zkConnect: "" + # Must match in-cluster Solr Service DNS for this release/namespace (edit if your Helm release or ns differs). + solrHttpBase: "http://demo-dataverseup-solr.demo-dataverseup.svc.cluster.local:8983" + collection: dataverse + configSetName: dataverse_config + numShards: 1 + replicationFactor: 1 + confConfigMap: dataverse-solr-conf + existingSecret: "" + adminUser: "" + adminPassword: "" + resources: {} + +# Compose mounts **scripts/init.d/** at `/opt/payara/init.d`. Enable this to ship the same `*.sh` bundle via +# the chart ConfigMap (see **initdFromChart** in **charts/dataverseup/values.yaml**). Scripts run in lexical order; +# **012/013-minio** expect compose MinIO (no-op or skip if you use S3 only). **1001-webhooks** needs **scripts/triggers/** mounted at `/opt/payara/triggers` (Compose default). +# on disk unless you mount it separately. +initdFromChart: + enabled: true + +# Mount scripts/init.d/010-mailrelay-set.sh (compose-compatible). Requires system_email (and related env) to do work. +mail: + enabled: true + +# Payara/Dataverse — external Postgres + Solr. Create DB + role on the server first (quoted names if hyphenated); +# see ops/kubernetes-deploy-checklist.md — DB name here must exist on the server (e.g. demo-dataverseup). +extraEnvVars: + - name: DATAVERSE_DB_HOST + value: postgres-postgresql.postgres.svc.cluster.local + - name: DATAVERSE_DB_USER + value: "demo-dataverseup" + - name: DATAVERSE_DB_PASSWORD + value: $DB_PASSWORD + - name: DATAVERSE_DB_NAME + value: "demo-dataverseup" + - name: POSTGRES_SERVER + value: postgres-postgresql.postgres.svc.cluster.local + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DATABASE + value: "demo-dataverseup" + - name: POSTGRES_USER + value: "demo-dataverseup" + - name: POSTGRES_PASSWORD + value: $DB_PASSWORD + - name: PGPASSWORD + value: $DB_PASSWORD + # Uppercase SOLR_* / DATAVERSE_SOLR_* are set by the chart when internalSolr.enabled (see templates/deployment.yaml). + # Duplicate keys here caused kubectl apply warnings ("hides previous definition"). Override only for external Solr. + - name: dataverse_solr_host + value: "demo-dataverseup-solr.demo-dataverseup.svc.cluster.local" + - name: dataverse_solr_port + value: "8983" + - name: dataverse_solr_core + value: dataverse + # In-cluster Dataverse HTTP base (for tooling that expects a full URL). If you mount compose-style scripts/init.d + # that does http://${DATAVERSE_URL}/api, use host:port only (no scheme) instead. + - name: DATAVERSE_URL + value: "http://demo-dataverseup.demo-dataverseup.svc.cluster.local:80" + # Official image (init_2_configure.sh): dataverse_* env → Payara create-system-properties (dots from underscores). + # Without these, JSF often cannot resolve the public URL and "/" shows Dataverse "Page Not Found". + - name: dataverse_siteUrl + value: "https://demo-dataverseup.notch8.cloud" + - name: dataverse_fqdn + value: "demo-dataverseup" + - name: DATAVERSE_SERVICE_HOST + value: "demo-dataverseup" + - name: hostname + value: "demo-dataverseup" + - name: JVM_OPTS + value: "-Xmx2g -Xms2g" + # Dataverse 6.2+ sends mail via MicroProfile dataverse.mail.* (env DATAVERSE_MAIL_*), not only :SystemEmail / + # Payara JavaMail. Without these, verification/password mail may never send. See: + # https://guides.dataverse.org/en/latest/installation/config.html#smtp-email-configuration + - name: DATAVERSE_MAIL_SYSTEM_EMAIL + value: "support@notch8.com" + - name: DATAVERSE_MAIL_SUPPORT_EMAIL + value: "support@notch8.com" + - name: DATAVERSE_MAIL_DEBUG + value: "false" + - name: DATAVERSE_MAIL_MTA_HOST + value: "smtp.sendgrid.net" + - name: DATAVERSE_MAIL_MTA_PORT + value: "587" + - name: DATAVERSE_MAIL_MTA_AUTH + value: "true" + - name: DATAVERSE_MAIL_MTA_USER + value: "apikey" + - name: DATAVERSE_MAIL_MTA_PASSWORD + value: "${SMTP_PASSWORD}" + - name: DATAVERSE_MAIL_MTA_STARTTLS_ENABLE + value: "true" + - name: DATAVERSE_PID_PROVIDERS + value: demo + - name: DATAVERSE_PID_DEFAULT_PROVIDER + value: demo + - name: DATAVERSE_PID_DEMO_TYPE + value: FAKE + - name: DATAVERSE_PID_DEMO_LABEL + value: demo + - name: DATAVERSE_PID_DEMO_AUTHORITY + value: "10.5072" + - name: DATAVERSE_PID_DEMO_SHOULDER + value: FK2/ + - name: INIT_SCRIPTS_FOLDER + value: /opt/payara/init.d + # Payara JavaMail + :SystemEmail (scripts/init.d/010-mailrelay-set.sh). + # Literal SMTP_* (override via GitHub Environment if you reintroduce envsubst for these keys). + - name: SMTP_ADDRESS + value: "smtp.sendgrid.net" + - name: SMTP_USER_NAME + value: "apikey" + - name: SMTP_TYPE + value: "plain" + - name: SMTP_STARTTLS + value: "true" + - name: SMTP_PORT + value: "587" + - name: SMTP_ENABLED + value: "true" + - name: SMTP_DOMAIN + value: "notch8.com" + # Script reads these names (aligned with SMTP_* above). Both support@notch8.com: :SystemEmail + JavaMail From. + - name: system_email + value: "support@notch8.com" + - name: mailhost + value: "smtp.sendgrid.net" + - name: mailuser + value: "apikey" + - name: no_reply_email + value: "support@notch8.com" + - name: smtp_password + value: "${SMTP_PASSWORD}" + - name: smtp_port + value: "587" + - name: socket_port + value: "587" + - name: smtp_auth + value: "true" + - name: smtp_starttls + value: "true" + - name: smtp_type + value: "plain" + - name: smtp_enabled + value: "true" + +# gdcc/configbaker after install (post-install hook). Same tag as image.tag above. +# **mode: compose** — same order as compose: configbaker → **apply-branding.sh** (reads **branding/branding.env** +# baked into chart) → optional **seed-content** (fixtures). Does not run on plain `helm upgrade` when helmHook=true. +# Re-apply branding after changing **branding/**: use workflow "Run bootstrap job" with hook disabled, or +# **postUpgradeBrandingSeedJob** + admin API token Secret (see **charts/dataverseup/values.yaml**). +# If you wiped Postgres later, use GitHub Deploy → "Run bootstrap job" or apply a Job by hand. +# +# **existingAdminApiTokenSecret** is optional on first install (chart uses optional secretKeyRef). Create the +# Secret after you have a superuser API token so re-runs / already-bootstrapped DB work: +# kubectl -n demo-dataverseup create secret generic dataverse-admin-api-token --from-literal=token=YOUR_UUID +bootstrapJob: + enabled: true + mode: compose + helmHook: true + timeout: 20m + compose: + waitMaxSeconds: 900 + waitSleepSeconds: 5 + seed: true + existingAdminApiTokenSecret: dataverse-admin-api-token + adminApiTokenSecretKey: token diff --git a/scripts/apply-branding.sh b/scripts/apply-branding.sh index 545e416..669a37a 100755 --- a/scripts/apply-branding.sh +++ b/scripts/apply-branding.sh @@ -42,17 +42,25 @@ if [ -n "${FOOTER_COPYRIGHT:-}" ]; then esac fi +# Path segment must be URL-encoded (at least ':') so proxies like nginx do not mishandle +# /api/admin/settings/:LogoCustomizationFile as a bogus port or split path. +admin_setting_path() { + printf '%s' "$1" | sed -e 's/%/%25/g' -e 's/:/%3A/g' -e 's/ /%20/g' +} + curl_put_setting() { _name="$1" _val="$2" if [ -z "$_val" ]; then return 0 fi + _path_seg=$(admin_setting_path "$_name") printf '%s' "apply-branding: PUT ${_name}\n" >&2 - _code=$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ + _code=$(curl -sS -g -o /dev/null -w "%{http_code}" -X PUT \ -H "X-Dataverse-key: ${TOKEN}" \ + -H "Content-Type: text/plain; charset=UTF-8" \ --data-binary "$_val" \ - "${API}/admin/settings/${_name}") + "${API}/admin/settings/${_path_seg}") if [ "$_code" != "200" ] && [ "$_code" != "204" ]; then echo "apply-branding: WARNING ${_name} -> HTTP ${_code} (check admin user API token in secrets/api/key)" >&2 fi diff --git a/init.d/006-s3-aws-storage.sh b/scripts/init.d/006-s3-aws-storage.sh similarity index 78% rename from init.d/006-s3-aws-storage.sh rename to scripts/init.d/006-s3-aws-storage.sh index beee273..2e2afa0 100644 --- a/init.d/006-s3-aws-storage.sh +++ b/scripts/init.d/006-s3-aws-storage.sh @@ -37,14 +37,10 @@ if [ -n "${aws_bucket_name:-}" ]; then | grep -v -E '^create-system-properties dataverse\.files\.S3\.' \ | grep -v -E '^deploy ' > "$_pb_pre" || true grep -E '^deploy ' "$POSTBOOT_COMMANDS_FILE" > "$_pb_dep" || true - _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') - # With custom-endpoint-url set, Dataverse's S3AccessIO uses JVM key custom-endpoint-region for SigV4. - # If unset, upstream defaults to the literal string "dataverse" → S3 400 "region 'dataverse' is wrong". + # Amazon S3: leave custom-endpoint-url unset so the AWS SDK uses default resolution (virtual-hosted buckets, + # correct SigV4). Setting a regional URL here often breaks uploads with opaque "Failed to save the content" errors. + # MinIO / S3-compatible: set aws_endpoint_url (Helm awsS3.endpointUrl) to your service base URL. _s3_reg="${aws_s3_region:-${AWS_REGION:-}}" - if [ -z "${_s3_reg}" ]; then - echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when using custom-endpoint-url" >&2 - return 1 - fi { cat "$_pb_pre" echo "create-system-properties dataverse.files.S3.type=s3" @@ -55,8 +51,15 @@ if [ -n "${aws_bucket_name:-}" ]; then echo "create-system-properties dataverse.files.S3.connection-pool-size=4096" echo "create-system-properties dataverse.files.storage-driver-id=S3" echo "create-system-properties dataverse.files.S3.profile=${aws_s3_profile}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" - echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" + if [ -n "${aws_endpoint_url:-}" ]; then + if [ -z "${_s3_reg}" ]; then + echo "006-s3-aws-storage: set aws_s3_region (Helm awsS3.region) or AWS_REGION when aws_endpoint_url is set" >&2 + return 1 + fi + _ep=$(printf '%s' "${aws_endpoint_url}" | sed -e 's/:/\\\:/g') + echo "create-system-properties dataverse.files.S3.custom-endpoint-url=${_ep}" + echo "create-system-properties dataverse.files.S3.custom-endpoint-region=${_s3_reg}" + fi cat "$_pb_dep" } > "$POSTBOOT_COMMANDS_FILE" trap - EXIT diff --git a/init.d/01-persistent-id.sh b/scripts/init.d/01-persistent-id.sh similarity index 100% rename from init.d/01-persistent-id.sh rename to scripts/init.d/01-persistent-id.sh diff --git a/init.d/010-languages.sh b/scripts/init.d/010-languages.sh similarity index 100% rename from init.d/010-languages.sh rename to scripts/init.d/010-languages.sh diff --git a/init.d/010-mailrelay-set.sh b/scripts/init.d/010-mailrelay-set.sh similarity index 83% rename from init.d/010-mailrelay-set.sh rename to scripts/init.d/010-mailrelay-set.sh index 343900d..5fe74c7 100644 --- a/init.d/010-mailrelay-set.sh +++ b/scripts/init.d/010-mailrelay-set.sh @@ -1,9 +1,10 @@ #!/bin/bash -# Setup mail relay +# Setup mail relay (Compose mount scripts/init.d + Helm ConfigMap when values.mail.enabled). # https://guides.dataverse.org/en/latest/developers/troubleshooting.html # -# smtp_enabled / smtp_type=plain — same behavior as charts/demo-dataverse/files/010-mailrelay-set.sh +# smtp_enabled: false|0|no skips. smtp_type=plain can imply SMTP AUTH when smtp_auth is unset. +# smtp_auth / smtp_starttls: true|1|yes when set explicitly. case "${smtp_enabled}" in false|0|no|NO|False) exit 0 ;; esac diff --git a/init.d/011-local-storage.sh b/scripts/init.d/011-local-storage.sh similarity index 100% rename from init.d/011-local-storage.sh rename to scripts/init.d/011-local-storage.sh diff --git a/init.d/012-minio-bucket1.sh b/scripts/init.d/012-minio-bucket1.sh similarity index 100% rename from init.d/012-minio-bucket1.sh rename to scripts/init.d/012-minio-bucket1.sh diff --git a/init.d/013-minio-bucket2.sh b/scripts/init.d/013-minio-bucket2.sh similarity index 100% rename from init.d/013-minio-bucket2.sh rename to scripts/init.d/013-minio-bucket2.sh diff --git a/init.d/02-controlled-voc.sh b/scripts/init.d/02-controlled-voc.sh similarity index 100% rename from init.d/02-controlled-voc.sh rename to scripts/init.d/02-controlled-voc.sh diff --git a/init.d/03-doi-set.sh b/scripts/init.d/03-doi-set.sh similarity index 100% rename from init.d/03-doi-set.sh rename to scripts/init.d/03-doi-set.sh diff --git a/init.d/04-setdomain.sh b/scripts/init.d/04-setdomain.sh similarity index 100% rename from init.d/04-setdomain.sh rename to scripts/init.d/04-setdomain.sh diff --git a/init.d/05-reindex.sh b/scripts/init.d/05-reindex.sh similarity index 100% rename from init.d/05-reindex.sh rename to scripts/init.d/05-reindex.sh diff --git a/init.d/07-previewers.sh b/scripts/init.d/07-previewers.sh similarity index 100% rename from init.d/07-previewers.sh rename to scripts/init.d/07-previewers.sh diff --git a/init.d/08-federated-login.sh b/scripts/init.d/08-federated-login.sh similarity index 100% rename from init.d/08-federated-login.sh rename to scripts/init.d/08-federated-login.sh diff --git a/init.d/1001-webhooks.sh b/scripts/init.d/1001-webhooks.sh similarity index 100% rename from init.d/1001-webhooks.sh rename to scripts/init.d/1001-webhooks.sh diff --git a/init.d/1002-custom-metadata.sh b/scripts/init.d/1002-custom-metadata.sh similarity index 93% rename from init.d/1002-custom-metadata.sh rename to scripts/init.d/1002-custom-metadata.sh index 6f4e22b..46c976b 100755 --- a/init.d/1002-custom-metadata.sh +++ b/scripts/init.d/1002-custom-metadata.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -# Solr field-update scripts are vendored under init.d/vendor-solr/ (from IQSS Dataverse v5.x release assets; stable helpers for schema merge + Solr MDB update). +# Solr field-update scripts are vendored under vendor-solr/ next to this script (repo: scripts/init.d/vendor-solr/; container: $HOME_DIR/init.d/vendor-solr/). # Avoids runtime version skew against the running Dataverse image. if [ "${CLARIN:-}" ]; then diff --git a/init.d/vendor-solr/update-fields.sh b/scripts/init.d/vendor-solr/update-fields.sh similarity index 100% rename from init.d/vendor-solr/update-fields.sh rename to scripts/init.d/vendor-solr/update-fields.sh diff --git a/init.d/vendor-solr/updateSchemaMDB.sh b/scripts/init.d/vendor-solr/updateSchemaMDB.sh similarity index 100% rename from init.d/vendor-solr/updateSchemaMDB.sh rename to scripts/init.d/vendor-solr/updateSchemaMDB.sh diff --git a/scripts/k8s-bootstrap-chain.sh b/scripts/k8s-bootstrap-chain.sh new file mode 100755 index 0000000..d2ac020 --- /dev/null +++ b/scripts/k8s-bootstrap-chain.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Helm Job: same order as docker compose dev_bootstrap → dev_branding → dev_seed. +# Runs inside gdcc/configbaker (bootstrap.sh) then applies branding + seed via curl scripts. +set -euo pipefail + +# Bump when token / configbaker behavior changes (grep logs for this to confirm the cluster mounted an updated ConfigMap). +CHAIN_SCRIPT_REVISION=4 +echo "k8s-bootstrap-chain: chain script revision=${CHAIN_SCRIPT_REVISION}" >&2 + +# Helm mounts our chain scripts here; must NOT use /scripts — that shadows gdcc/configbaker's /scripts/bootstrap.sh. +CHAIN_SCRIPTS="${BOOTSTRAP_CHAIN_SCRIPT_DIR:-/bootstrap-chain}" +CONFIGBAKER_BOOTSTRAP="${CONFIGBAKER_BOOTSTRAP_SH:-/scripts/bootstrap.sh}" + +SECRETS_DIR="${DATAVERSE_BOOTSTRAP_SECRETS_DIR:-/secrets}" +TOKEN_FILE="${DATAVERSE_BOOTSTRAP_ENV_FILE:-${SECRETS_DIR}/api/bootstrap.env}" +WAIT_MAX="${BOOTSTRAP_CHAIN_WAIT_MAX_SECONDS:-900}" +SLEEP="${BOOTSTRAP_CHAIN_WAIT_SLEEP:-5}" + +DATAVERSE_INTERNAL_URL="${DATAVERSE_INTERNAL_URL:-}" +if [[ -z "${DATAVERSE_INTERNAL_URL}" ]]; then + echo "k8s-bootstrap-chain: DATAVERSE_INTERNAL_URL is required" >&2 + exit 1 +fi + +if [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: DATAVERSE_API_TOKEN is set from the environment (length=${#DATAVERSE_API_TOKEN})" >&2 +else + echo "k8s-bootstrap-chain: DATAVERSE_API_TOKEN is unset (first install is OK; already-bootstrapped needs a Secret — see values bootstrapJob.compose.existingAdminApiTokenSecret)" >&2 +fi + +try_version() { + curl -sf --max-time 15 "${DATAVERSE_INTERNAL_URL%/}/api/info/version" >/dev/null 2>&1 +} + +echo "k8s-bootstrap-chain: waiting for Dataverse at ${DATAVERSE_INTERNAL_URL} (max ${WAIT_MAX}s) ..." >&2 +elapsed=0 +while [[ "${elapsed}" -lt "${WAIT_MAX}" ]]; do + if try_version; then + echo "k8s-bootstrap-chain: Dataverse API is up" >&2 + break + fi + sleep "${SLEEP}" + elapsed=$((elapsed + SLEEP)) +done +if ! try_version; then + echo "k8s-bootstrap-chain: timeout — Dataverse not reachable" >&2 + exit 1 +fi + +write_api_key_from_token() { + local tok="$1" + mkdir -p "${SECRETS_DIR}/api" + umask 077 + printf '%s\n' "${tok}" >"${SECRETS_DIR}/api/key.tmp" + mv "${SECRETS_DIR}/api/key.tmp" "${SECRETS_DIR}/api/key" + export DATAVERSE_API_TOKEN="${tok}" +} + +# Skip configbaker (e.g. post-upgrade Helm hook): supply DATAVERSE_API_TOKEN from env / secretRef. +if [[ "${BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP:-}" == 1 ]]; then + if [[ -z "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: BOOTSTRAP_CHAIN_SKIP_BOOTSTRAP=1 requires DATAVERSE_API_TOKEN" >&2 + exit 2 + fi + echo "k8s-bootstrap-chain: skipping bootstrap.sh (using existing admin API token)" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" +else + # Secret (optional ref) + non-empty token: skip configbaker — it only prints "already bootstrapped" and leaves no API_TOKEN in bootstrap.env. + if [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: skipping configbaker — using DATAVERSE_API_TOKEN from environment" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" + else + mkdir -p "$(dirname "${TOKEN_FILE}")" + if [[ -d "${TOKEN_FILE}" ]]; then + echo "k8s-bootstrap-chain: ${TOKEN_FILE} is a directory" >&2 + exit 3 + fi + umask 077 + [[ -f "${TOKEN_FILE}" ]] || : >"${TOKEN_FILE}" + + echo "k8s-bootstrap-chain: running configbaker (${CONFIGBAKER_BOOTSTRAP} -e ${TOKEN_FILE} dev) ..." >&2 + if [[ ! -x "${CONFIGBAKER_BOOTSTRAP}" ]]; then + echo "k8s-bootstrap-chain: ${CONFIGBAKER_BOOTSTRAP} missing or not executable (is /scripts shadowed by a volume mount?)" >&2 + exit 127 + fi + "${CONFIGBAKER_BOOTSTRAP}" -e "${TOKEN_FILE}" dev + + # Kubernetes may inject DATAVERSE_API_TOKEN via secretKeyRef; configbaker's bootstrap.env can still + # assign empty vars — sourcing must not wipe that token. + DATAVERSE_API_TOKEN_PRE_SOURCE="${DATAVERSE_API_TOKEN:-}" + + # shellcheck disable=SC1090 + set +e + [[ -f "${TOKEN_FILE}" ]] && source "${TOKEN_FILE}" + set -e + + if [[ -n "${API_TOKEN:-}" ]]; then + write_api_key_from_token "${API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE} (configbaker often skips when already bootstrapped); using DATAVERSE_API_TOKEN from environment" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN}" + elif [[ -n "${DATAVERSE_API_TOKEN_PRE_SOURCE:-}" ]]; then + echo "k8s-bootstrap-chain: no API_TOKEN in ${TOKEN_FILE}; bootstrap.env cleared DATAVERSE_API_TOKEN — using token from environment (e.g. Secret) before source" >&2 + write_api_key_from_token "${DATAVERSE_API_TOKEN_PRE_SOURCE}" + else + echo "k8s-bootstrap-chain: no API_TOKEN after configbaker and no usable DATAVERSE_API_TOKEN." >&2 + echo "k8s-bootstrap-chain: fix: kubectl create secret generic ... --from-literal=token=SUPERUSER_UUID and set bootstrapJob.compose.existingAdminApiTokenSecret (see ops/demo-deploy.tmpl.yaml)." >&2 + echo "k8s-bootstrap-chain: if startup logs lack \"chain script revision=${CHAIN_SCRIPT_REVISION}\", the Helm bootstrap-chain ConfigMap is stale — helm upgrade the release or kubectl apply the rendered templates/bootstrap-chain-configmap.yaml manifest." >&2 + exit 2 + fi + fi +fi + +export BRANDING_ENV_PATH="${BRANDING_ENV_PATH:-/config/branding.env}" +if [[ -f "${BRANDING_ENV_PATH}" ]]; then + echo "k8s-bootstrap-chain: apply-branding ..." >&2 + export DATAVERSE_INTERNAL_URL + /bin/sh "${CHAIN_SCRIPTS}/apply-branding.sh" +else + echo "k8s-bootstrap-chain: skip branding (no ${BRANDING_ENV_PATH})" >&2 +fi + +layout_seed_from_flat_mount() { + # ConfigMap keys cannot contain '/'; flat keys may be mounted under /seed-flat. + [[ -d /seed-flat ]] || return 0 + mkdir -p /fixtures/seed/files + for f in demo-collection.json dataset-images.json dataset-tabular.json; do + [[ -f "/seed-flat/${f}" ]] && cp "/seed-flat/${f}" "/fixtures/seed/${f}" + done + for pair in "files_1x1.png:1x1.png" "files_badge.svg:badge.svg" "files_readme.txt:readme.txt" "files_sample.csv:sample.csv"; do + key="${pair%%:*}" + name="${pair##*:}" + [[ -f "/seed-flat/${key}" ]] && cp "/seed-flat/${key}" "/fixtures/seed/files/${name}" + done +} + +if [[ "${BOOTSTRAP_CHAIN_SEED:-1}" == "1" ]]; then + layout_seed_from_flat_mount +fi + +if [[ "${BOOTSTRAP_CHAIN_SEED:-1}" == "1" ]] && [[ -f /fixtures/seed/demo-collection.json ]]; then + echo "k8s-bootstrap-chain: seed-content ..." >&2 + export SEED_FIXTURE="${SEED_FIXTURE:-/fixtures/seed/demo-collection.json}" + export SEED_ROOT="${SEED_ROOT:-/fixtures/seed}" + export SEED_PARENT_ALIAS="${SEED_PARENT_ALIAS:-root}" + /bin/sh "${CHAIN_SCRIPTS}/seed-content.sh" +else + echo "k8s-bootstrap-chain: skip seed (BOOTSTRAP_CHAIN_SEED=${BOOTSTRAP_CHAIN_SEED:-} or no fixtures)" >&2 +fi + +echo "k8s-bootstrap-chain: done" >&2 diff --git a/scripts/k8s/ensure-solr-conf-configmap.sh b/scripts/k8s/ensure-solr-conf-configmap.sh new file mode 100755 index 0000000..aaaf8ad --- /dev/null +++ b/scripts/k8s/ensure-solr-conf-configmap.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Build IQSS Dataverse Solr config (same layout as test-k3d-deploy/k3d/install.sh) and apply ConfigMap +# with key solr-conf.tgz — required by the chart's internal Solr init container. +# +# Uses curl + tar only (no Docker). Run after kubectl context points at the target cluster. +# +# Usage: ./scripts/k8s/ensure-solr-conf-configmap.sh NAMESPACE +# Env: +# DV_REF IQSS Dataverse tag (default: v6.10.1) +# SOLR_DIST_VERSION Apache Solr release for _default configset extras (default: 9.10.1) +# CONFIGMAP_NAME (default: dataverse-solr-conf) +set -eu + +NAMESPACE="${1:-}" +if [ -z "$NAMESPACE" ]; then + echo "usage: $0 NAMESPACE" >&2 + exit 1 +fi + +DV_REF="${DV_REF:-v6.10.1}" +SOLR_DIST_VERSION="${SOLR_DIST_VERSION:-9.10.1}" +CONFIGMAP_NAME="${CONFIGMAP_NAME:-dataverse-solr-conf}" + +for cmd in kubectl curl tar; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "missing required command: $cmd" >&2 + exit 1 + fi +done + +TMP="$(mktemp -d)" +cleanup() { + rm -rf "$TMP" +} +trap cleanup EXIT + +echo "Fetching IQSS Dataverse ${DV_REF} (conf/solr) ..." +curl -fsSL "https://codeload.github.com/IQSS/dataverse/tar.gz/${DV_REF}" -o "$TMP/dv.tgz" +tar -xzf "$TMP/dv.tgz" -C "$TMP" +TOP="$(find "$TMP" -maxdepth 1 -type d -name 'dataverse-*' | head -1)" +if [ -z "$TOP" ] || [ ! -d "$TOP/conf/solr" ]; then + echo "Could not find dataverse-*/conf/solr in archive (try DV_REF=develop or a valid tag)." >&2 + exit 1 +fi + +STAGE="${TMP}/solr-stage" +mkdir -p "${STAGE}" +cp -a "${TOP}/conf/solr/." "${STAGE}/" + +SPREFIX="solr-${SOLR_DIST_VERSION}/server/solr/configsets/_default/conf" +SOLR_URL="https://archive.apache.org/dist/solr/solr/${SOLR_DIST_VERSION}/solr-${SOLR_DIST_VERSION}.tgz" +echo "Fetching Solr ${SOLR_DIST_VERSION} _default conf companions from ${SOLR_URL} ..." +curl -fsSL "${SOLR_URL}" -o "$TMP/apache-solr.tgz" +tar -xzf "$TMP/apache-solr.tgz" -C "$TMP" \ + "${SPREFIX}/lang" \ + "${SPREFIX}/protwords.txt" \ + "${SPREFIX}/stopwords.txt" \ + "${SPREFIX}/synonyms.txt" + +CONFROOT="${TMP}/${SPREFIX}" +cp -a "${CONFROOT}/lang" "${STAGE}/" +for f in protwords.txt stopwords.txt synonyms.txt; do + cp -a "${CONFROOT}/${f}" "${STAGE}/${f}" +done + +SOLR_TGZ="${TMP}/solr-conf.tgz" +tar -czf "${SOLR_TGZ}" -C "${STAGE}" . +echo "Packaged Solr conf ($(du -h "${SOLR_TGZ}" | cut -f1)) → ConfigMap ${CONFIGMAP_NAME} key solr-conf.tgz" + +kubectl -n "${NAMESPACE}" create configmap "${CONFIGMAP_NAME}" \ + --from-file=solr-conf.tgz="${SOLR_TGZ}" \ + --dry-run=client -o yaml | kubectl apply -f - + +echo "Applied ConfigMap ${CONFIGMAP_NAME} in namespace ${NAMESPACE}" diff --git a/scripts/seed-content.sh b/scripts/seed-content.sh index 04a02ad..c0588bc 100755 --- a/scripts/seed-content.sh +++ b/scripts/seed-content.sh @@ -96,12 +96,33 @@ upload_file() { _dir=$3 _tab=$4 _enc=$(encode_pid "$_pid") - _json=$(printf '{"description":"DataverseUp seed","directoryLabel":"%s","restrict":"false","tabIngest":"%s"}' "$_dir" "$_tab") - curl -fsS --max-time 300 --globoff -X POST \ + # Booleans must be JSON true/false (not quoted strings); include categories per Native API examples. + _json=$(printf '{"description":"DataverseUp seed","directoryLabel":"%s","restrict":false,"tabIngest":%s,"categories":["Data"]}' "$_dir" "$_tab") + _resp=$(curl -sS --max-time 300 --globoff -X POST \ -H "X-Dataverse-key: ${TOKEN}" \ -F "file=@${_path}" \ -F "jsonData=${_json}" \ - "${API}/datasets/:persistentId/add?persistentId=${_enc}" + -w "\n%{http_code}" \ + "${API}/datasets/:persistentId/add?persistentId=${_enc}" || printf '%s\n' "000") + _code=$(printf '%s\n' "$_resp" | tail -n 1) + _body=$(printf '%s\n' "$_resp" | sed '$d') + case "$_code" in + 200|201|204) + return 0 + ;; + *) + echo "seed-content: upload $(basename "$_path") failed HTTP ${_code}" >&2 + printf '%s\n' "$_body" >&2 + case "$_body" in + *Failed*to*save*the*content*) + echo "seed-content: hint: Dataverse accepted the upload but could not write to file storage." >&2 + echo "seed-content: hint: With awsS3.enabled, check IAM (s3:PutObject/GetObject/DeleteObject/ListBucket on the bucket), bucket name/region vs values, and that pods were restarted after creating aws-s3-credentials. See docs/DEPLOYMENT.md (S3 troubleshooting)." >&2 + echo "seed-content: hint: Inspect the Dataverse pod logs for AWS SDK errors (AccessDenied, NoSuchBucket, etc.)." >&2 + ;; + esac + exit 1 + ;; + esac } # Datasets cannot be published while their host collection is still unpublished (API often returns 403). diff --git a/scripts/solr-init-k8s.sh b/scripts/solr-init-k8s.sh new file mode 100755 index 0000000..b84ddbf --- /dev/null +++ b/scripts/solr-init-k8s.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# Apply Dataverse Solr conf ConfigMap to a namespace and optionally restart in-chart workloads so +# init containers / Solr pick up new conf (e.g. after IQSS schema changes or first-time bootstrap). +# +# Requires: kubectl, curl, tar (same as scripts/k8s/ensure-solr-conf-configmap.sh). +# +# Usage (from repo root): +# ./scripts/solr-init-k8s.sh NAMESPACE HELM_RELEASE_NAME +# +# Env: +# SOLR_APPLY_CM=true|false Apply ConfigMap (default: true) +# SOLR_RESTART_DEPLOYMENTS=true|false Rollout restart matching Deployments (default: true) +# CHART_APP_NAME app.kubernetes.io/name base label (default: dataverseup) +# DV_REF, SOLR_DIST_VERSION, CONFIGMAP_NAME — forwarded to ensure-solr-conf-configmap.sh +# SOLR_ROLLOUT_TIMEOUT e.g. 5m (default: 8m) +set -eu + +NS="${1:-}" +REL="${2:-}" +if [ -z "$NS" ] || [ -z "$REL" ]; then + echo "usage: $0 NAMESPACE HELM_RELEASE_NAME" >&2 + echo " example: $0 demo-dataverseup demo-dataverseup" >&2 + exit 1 +fi + +SOLR_APPLY_CM="${SOLR_APPLY_CM:-true}" +SOLR_RESTART_DEPLOYMENTS="${SOLR_RESTART_DEPLOYMENTS:-true}" +CHART_APP_NAME="${CHART_APP_NAME:-${HELM_APP_NAME:-dataverseup}}" +SOLR_ROLLOUT_TIMEOUT="${SOLR_ROLLOUT_TIMEOUT:-8m}" + +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +ENSURE="${ROOT}/scripts/k8s/ensure-solr-conf-configmap.sh" + +if [ -f "$ENSURE" ] && [ ! -x "$ENSURE" ]; then + chmod +x "$ENSURE" || true +fi +if [ ! -f "$ENSURE" ]; then + echo "missing ${ENSURE}" >&2 + exit 1 +fi + +command -v kubectl >/dev/null 2>&1 || { + echo "kubectl is required" >&2 + exit 1 +} + +_truthy() { + case "$1" in + 1 | true | TRUE | yes | YES) return 0 ;; + *) return 1 ;; + esac +} + +if _truthy "$SOLR_APPLY_CM"; then + echo "=== Solr: applying ConfigMap (${CONFIGMAP_NAME:-dataverse-solr-conf}) ===" + "$ENSURE" "$NS" +else + echo "=== Solr: skipping ConfigMap apply (SOLR_APPLY_CM=false) ===" +fi + +if ! _truthy "$SOLR_RESTART_DEPLOYMENTS"; then + echo "=== Solr: skipping rollout restarts (SOLR_RESTART_DEPLOYMENTS=false) ===" + exit 0 +fi + +SOLR_LABEL="app.kubernetes.io/instance=${REL},app.kubernetes.io/name=${CHART_APP_NAME}-solr" +APP_LABEL="app.kubernetes.io/instance=${REL},app.kubernetes.io/name=${CHART_APP_NAME}" + +restart_labeled() { + _label="$1" + _kind="$2" + kubectl -n "$NS" get deploy -l "$_label" -o name 2>/dev/null | while read -r res; do + [ -z "$res" ] && continue + echo "=== Solr: rollout restart $_kind ($res) ===" + kubectl -n "$NS" rollout restart "$res" + kubectl -n "$NS" rollout status "$res" --timeout="$SOLR_ROLLOUT_TIMEOUT" + done +} + +# Solr first (Dataverse init container waits for core ping). +restart_labeled "$SOLR_LABEL" "internal Solr" +restart_labeled "$APP_LABEL" "Dataverse" + +echo "=== Solr: done ===" diff --git a/config/update-fields.sh b/scripts/solr/update-fields.sh similarity index 100% rename from config/update-fields.sh rename to scripts/solr/update-fields.sh diff --git a/triggers/affiliations.sql b/scripts/triggers/affiliations.sql similarity index 100% rename from triggers/affiliations.sql rename to scripts/triggers/affiliations.sql diff --git a/triggers/external-service.sql b/scripts/triggers/external-service.sql similarity index 100% rename from triggers/external-service.sql rename to scripts/triggers/external-service.sql diff --git a/triggers/external-services.py b/scripts/triggers/external-services.py similarity index 100% rename from triggers/external-services.py rename to scripts/triggers/external-services.py diff --git a/triggers/lang-properties-convert.py b/scripts/triggers/lang-properties-convert.py similarity index 100% rename from triggers/lang-properties-convert.py rename to scripts/triggers/lang-properties-convert.py