diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/SKILL.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/SKILL.md new file mode 100644 index 000000000..5f2b8e0af --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/SKILL.md @@ -0,0 +1,44 @@ +--- +name: oci-devops-starter-maintainer +description: Maintain, extend, validate, package, and functionally test the OKE DevOps Starter Resource Manager stack. Use when changing its Terraform, Resource Manager schema, OCI DevOps repositories, build or deployment pipelines, Helm charts, cluster-administration workflow, documentation, or release archive. +--- + +# OKE DevOps Starter Maintainer + +Use this Skill only from the root of the OKE DevOps Starter repository. + +## Start Every Task + +1. Read `README.md` for the product contract and documentation map. +2. Read `AGENT.md` when it exists for local development history. Treat its environment identifiers as local context, never distributable content. +3. Inspect the working tree and preserve unrelated user changes. +4. Read only the references relevant to the task: + - Architecture, ownership, and source locations: `references/architecture.md` + - Invariants, security boundaries, and OCI constraints: `references/invariants.md` + - Local and functional verification: `references/testing.md` + +## Maintenance Workflow + +1. Trace the requested behavior from Resource Manager input through normalized locals, OCI resources, templates, seeded repository content, and documentation. +2. Modify the true source of the generated behavior. Do not patch generated output when a Terraform template or local generates it. +3. Preserve template ownership: OCI DevOps resources are starters, and repository seed operations must not overwrite user-owned paths. +4. Keep changes entity-generic. Test at least two applications or components mentally whenever changing `for_each` maps, names, paths, parameters, tags, or pipeline wiring. +5. Add or update regression tests for structural contracts and past OCI provider failures. +6. Run `bash .agents/skills/oci-devops-starter-maintainer/scripts/validate.sh`. +7. When packaging is requested, run `./update.sh`, inspect the archive, and confirm `.agents`, `AGENT.md`, local state, credentials, logs, and ad hoc scripts are absent. +8. Apply or functionally test live OCI resources only when requested. Prefer focused tests after narrow changes and end-to-end tests after workflow changes. + +## Editing Rules + +- Keep Terraform authoritative for stable resource wiring and infrastructure, while preserving configured lifecycle ignores for user-customizable DevOps templates. +- Keep `schema.yaml`, `variables.tf`, validation checks, locals, outputs, and documentation synchronized when changing an input. +- Keep build specifications readable as macro steps; put complex shell behavior in named repository scripts. +- Use lowercase pipeline parameters and exported variables unless an OCI-defined variable requires another form. +- Never embed tenancy-specific OCIDs, usernames, auth tokens, secret values, or test-environment history in distributable files. +- Do not introduce OCI resources managed through `local-exec` or OCI CLI. Existing `null_resource` use is limited to add-only Git repository seeding. +- Do not assume pre-prod and prod are different physical clusters; the stack supports selecting the same cluster for both. +- Do not weaken production approvals or immutable image/chart/version behavior without explicit direction. + +## Completion Report + +State which product behavior changed, which checks ran, whether an archive or stack was updated, and any live functional test or cleanup that remains. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/agents/openai.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/agents/openai.yaml new file mode 100644 index 000000000..8778f1ffb --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "OKE DevOps Starter Maintainer" + short_description: "Safely maintain and test the OCI DevOps starter" + default_prompt: "Use $oci-devops-starter-maintainer to make and validate this stack change safely." + +policy: + allow_implicit_invocation: true diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/architecture.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/architecture.md new file mode 100644 index 000000000..13e8d1b12 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/architecture.md @@ -0,0 +1,62 @@ +# Architecture And Source Ownership + +## Product Paths + +The stack creates two related workflows: + +- Developer delivery: application baselines plus independently built and released components. +- Cluster administration: cluster tools, supplemental namespace resources, and cluster-wide baseline resources. + +Read `docs/architecture.md`, then use the task-specific guides linked from `README.md`. + +## Input To Resource Flow + +```text +schema.yaml + -> variables.tf and checks.tf + -> root locals.tf + -> main.tf module inputs + -> modules/devops/locals.tf normalized entity maps + -> Terraform repositories, artifacts, pipelines, stages, triggers, and environments + -> templates/*.tpl and repos/* starter content +``` + +When changing an input, inspect every layer in that path. `schema.yaml` controls the Resource Manager form but does not replace Terraform validation. + +## Source Map + +| Concern | Primary source locations | +| --- | --- | +| Resource Manager form | `schema.yaml` | +| Inputs and validation | `variables.tf`, `checks.tf`, `locals.tf` | +| Root composition and outputs | `main.tf`, `outputs.tf` | +| Entity normalization | `modules/devops/locals.tf` | +| Repositories and seeding | `modules/devops/repositories.tf`, `modules/devops/cluster_admin_repositories.tf`, `script/seed_repo.sh` | +| Build pipelines and triggers | `modules/devops/build_pipelines.tf`, `modules/devops/cluster_admin_build_pipelines.tf`, `modules/devops/triggers.tf` | +| Deployment pipelines | `modules/devops/deploy_pipelines.tf`, `modules/devops/cluster_admin_deploy_pipelines.tf` | +| Artifacts | `modules/devops/artifacts.tf`, `modules/devops/cluster_admin_artifacts.tf` | +| Build and command specifications | `templates/*.yaml.tpl` | +| Shared runtime scripts | `repos/pipelines/script`, `repos/cluster-admin/script` | +| Generated starter content | `repos/generated` plus `local_file` resources | +| Packaging | `update.sh` | +| Stack upload helper | `script/update_orm_stack.sh` | +| Regression tests | `tests/` | + +## Ownership Boundaries + +- Terraform creates stable OCI wiring and initial templates. +- Release archives retain lifecycle ignores on user-customizable OCI DevOps resources. +- Development archives remove only the exact release ownership blocks staged by `update.sh`. +- Git seed operations are add-only. They may create missing files and directories but must preserve an existing path. +- A custom component build-spec path receives the starter build spec only when missing and is never refreshed afterward. +- `catalog/tools.yaml` is generated from Resource Manager cluster-tool input; cluster-specific values and manifests remain administrator-owned. + +Read `docs/template-ownership.md` before changing lifecycle blocks, repository seed triggers, or generated files. + +## Entity Model + +- One shared `pipelines` repository. +- One chart repository and application baseline lifecycle per application. +- One source, PR, build, dev deployment, and release lifecycle per globally unique component. +- Optional shared `cluster-admin` repository and cluster-specific administration pipelines. +- Pre-prod and prod networking, environments, and approvals remain independently configured even when both target the same cluster. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/invariants.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/invariants.md new file mode 100644 index 000000000..2fae9f56d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/invariants.md @@ -0,0 +1,42 @@ +# Invariants And OCI Constraints + +## Developer Delivery + +- Application and component names are configurable; components are globally unique. +- Image paths use `//`. +- Umbrella charts use `/charts/` and component charts add `/`. +- Main builds produce only a 7-character Git SHA image tag and deploy development. +- Pull-request build specs are component-owned placeholders intended for language-specific tests. +- Application baselines and component charts have independent package and deployment lifecycles. +- Release builds promote an explicit semantic release-candidate tag without rebuilding the image. +- Production follows staging and approval, retags the RC image to the final version, deploys it, records status, and tags the released commit. +- Values artifacts are application/environment or cluster specific; image coordinates are pipeline parameters where required. + +## Cluster Administration + +- The feature is absent unless `enable_cluster_admin` is true. +- Public HTTP(S) and OCI Helm chart sources are supported and mirrored to the project chart prefix. +- Tool configuration forms a DAG through `depends_on`; orchestration, not arbitrary Terraform predecessor edges, enforces topological waves. +- Independent tools run in parallel. Tool Helm deployments happen before the final cluster-wide baseline because baseline objects may use tool-provided CRDs. +- Pre-prod begins without approval. Production requires approval before mutation. +- Values artifact versions equal the full configuration commit SHA. +- Supplemental namespaced resources may target only the configured tool namespace. +- Removing Git files does not prune live objects. Explicit decommission pipelines uninstall tools. + +## OCI DevOps Constraints + +- Deployment pipeline parameters may omit defaults. Build pipeline parameters cannot use empty defaults reliably, so derive build context from sources, stage metadata, and limited OCI lookups. +- Step-level environment assignments do not persist unless exported through the build specification contract; top-level build-spec environment values are available to all steps. +- OCI Helm `set_values` become chart values. Do not treat undocumented values such as `DRY_RUN=true` as stage control unless verified against the live service. +- Shell runners use OKE private endpoints and must use the target cluster worker subnet plus optional NSG. +- The OCI provider may reject empty strings on update even where the UI displays a blank default. Prefer omitted/null defaults when supported. +- Use `--stage-id` with `oci devops deploy-stage get`. +- Avoid changing predecessor graphs and resource keys in one apply when OCI resources still reference old stages. Use a staged migration when necessary. + +## Security And Distribution + +- Vault secret OCIDs are inputs, never embedded defaults for credentials. +- Secret values never enter Git, Terraform outputs, generated values, command specifications, or logs. +- Package only product Terraform, intentional templates/scripts, starter repository content, tests, and customer documentation. +- Exclude `.agents`, `AGENT.md`, `.git`, `.terraform`, state, tfvars, logs, credentials, local plans, generated archives, and private ad hoc deployment scripts. +- Inspect `docs/security.md` when changing IAM, Vault, networking, repository authentication, image/chart sources, or Kubernetes privileges. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/testing.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/testing.md new file mode 100644 index 000000000..1c812722e --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/references/testing.md @@ -0,0 +1,63 @@ +# Validation And Functional Testing + +## Local Validation + +Run the Skill validator from the repository root: + +```bash +bash .agents/skills/oci-devops-starter-maintainer/scripts/validate.sh +``` + +It checks Terraform formatting and validation, Python regressions, shell syntax, and every generated Helm chart found in the workspace. + +When `schema.yaml` changes, also validate it against the current OCI Resource Manager metadata JSON Schema supplied for the project. Confirm Resource Manager dependencies, visibility, required fields, regex validation, multiline JSON controls, and defaults in the graphical form after upload. + +## Archive Verification + +After `./update.sh`, list the archive and reject it if it contains any of: + +```text +.agents/ +AGENT.md +.git/ +.terraform/ +terraform.tfstate +*.tfvars +*.log +credentials or private test scripts +``` + +Test both packaging modes after changing lifecycle or packaging logic: + +```bash +./update.sh +STACK_DEVELOPMENT_MODE=true STACK_ZIP_PATH=/tmp/oke-devops-development.zip ./update.sh +``` + +Release mode must retain intended lifecycle ignores. Development mode must remove only the exact release ownership blocks and set the hidden development input. + +## Functional Test Selection + +Use the smallest test that proves the changed contract: + +| Change | Minimum live test | +| --- | --- | +| Resource Manager input or output | Upload/apply and inspect the graphical form plus outputs | +| Repository seeding | Apply twice; customize between applies and prove preservation | +| Component PR pipeline | Branch, PR, PR update, successful test run | +| Component build | Main commit, SHA7 image, dev deployment | +| Component release | RC promotion, staging, approval, prod, final image and Git tags | +| Application chart | Package baseline, noprod deploy, approval, prod deploy | +| Cluster tool | PR/build selection, mirror, target cluster deployment, health check | +| Cluster DAG | Independent parallel wave plus dependent tool wave | +| Decommission | Remove only the selected release/resources and preserve unrelated objects | + +## Live-Test Discipline + +1. Record resource IDs and commit hashes needed to correlate runs, but do not commit them to the Skill or distributable docs. +2. Inspect build/deployment logs and resulting OCIR, Git, Helm, and Kubernetes state. +3. When pre-prod and prod share a cluster, account for release-name and namespace collisions before testing. +4. Use direct `kubectl` only for verification and requested cleanup, not as a substitute for proving OCI DevOps execution. +5. Clean cluster-admin test charts and supplemental resources after successful functional tests unless the user asks to retain them. +6. Never approve, destroy, decommission, or clean production-scoped resources without explicit authorization. +7. Emit the configured completion notification only after the requested functional-test terminal condition is reached, and stop it immediately when requested. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/scripts/validate.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/scripts/validate.sh new file mode 100644 index 000000000..b9ab3e442 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/.agents/skills/oci-devops-starter-maintainer/scripts/validate.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -euo pipefail + +SKILL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="$(cd "${SKILL_DIR}/../../.." && pwd)" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +cd "$ROOT_DIR" + +echo "Checking Terraform formatting" +terraform fmt -check -recursive + +echo "Validating Terraform configuration" +terraform validate + +echo "Running regression tests" +if ! "$PYTHON_BIN" -c "import yaml" 2>/dev/null; then + echo "PyYAML is required by the regression tests; set PYTHON_BIN to a Python environment that provides it." >&2 + exit 1 +fi +"$PYTHON_BIN" -m unittest discover -s tests + +echo "Checking shell syntax" +while IFS= read -r -d '' script_path; do + bash -n "$script_path" +done < <(find repos/pipelines/script repos/cluster-admin/script script -type f -name '*.sh' -print0 2>/dev/null) + +echo "Linting generated Helm charts" +while IFS= read -r chart_file; do + helm lint "$(dirname "$chart_file")" +done < <(find repos/generated/charts -type f -name Chart.yaml -print 2>/dev/null | sort) + +echo "Maintainer validation succeeded" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/CHANGELOG.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/CHANGELOG.md new file mode 100644 index 000000000..482de8388 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/CHANGELOG.md @@ -0,0 +1,59 @@ +# Changelog + +All notable changes to the OKE DevOps Starter are documented in this file. + +This project follows [Semantic Versioning](https://semver.org/). + +## [1.0.0] - 2026-07-10 + +Initial public release of the OCI Resource Manager stack. + +### Application Delivery + +- Added support for multiple applications with multiple globally unique components from a concise JSON configuration. +- Added generated OCI Code repositories, protected main branches, pull-request validation pipelines, build pipelines, deployment pipelines, artifacts, triggers, and OCIR paths for every configured application and component. +- Added component-owned pull-request build specifications so teams can implement language-specific unit, integration, security, and quality checks. +- Added parameterless main builds that produce multi-architecture images tagged with the immutable 7-character Git commit SHA. +- Added automatic development deployment after a successful component build. +- Added release-candidate promotion with `skopeo`, staging deployment, production approval, final image promotion, production deployment, Helm status reporting, and final Git tagging. +- Separated application baseline charts from independently versioned and deployed component charts. +- Added per-application bootstrap pipelines that initialize application namespaces and OCIR pull secrets on pre-production and production clusters. +- Added cluster-specific application baseline values and environment-specific component values for development, staging, and production. + +### Cluster Administration + +- Added optional cluster administration behind the `enable_cluster_admin` feature flag. +- Added a dedicated `cluster-admin` repository for cluster tool configuration, values, supplemental namespaced resources, and cluster-wide baseline manifests. +- Added support for mirroring pinned public HTTP(S) and OCI Helm charts into OCIR. +- Added dependency-aware tool deployment using a configurable DAG, with independent tools deployed in parallel and dependent tools deployed in topological waves. +- Added immutable cluster values artifacts versioned by the full configuration commit SHA. +- Added selective deployments so a merged configuration change invokes only affected cluster and tool stages. +- Added matching pre-production and production orchestration, with approval required before production mutation. +- Added explicit tool decommission pipelines without automatic pruning of unrelated Kubernetes resources. + +### Resource Manager Experience + +- Added a guided Resource Manager form for OCI DevOps, applications, independent pre-production and production OKE targets, IAM, logging, notifications, and optional cluster administration. +- Added multiline, human-readable JSON defaults and validation for application/component and cluster-tool configuration. +- Added configurable DevOps project, logging, notification topic, Generic Artifact repository, IAM resource names, and component build-spec paths. +- Added structured application information outputs covering repositories, pipelines, environments, artifacts, OCIR paths, and suggested next actions. +- Added support for using one OKE cluster as both pre-production and production for evaluation environments. + +### Template Ownership And Safety + +- Added release-mode lifecycle protection so teams can customize generated OCI DevOps starter resources without later Resource Manager applies overwriting those changes. +- Added add-only Git repository seeding that preserves existing developer and administrator content. +- Added a separate hidden development packaging mode for stack maintainers to refresh template-managed resources during functional testing. +- Added Vault-backed OCIR pull credentials, private OKE endpoint access from configured worker subnets, optional NSGs, and scoped IAM policies. +- Added release packaging exclusions for local state, credentials, development artifacts, maintainer agent context, and private stack update helpers. + +### Documentation + +- Added a customer quickstart and a Deploy to Oracle Cloud entry point. +- Added separate developer and cluster-operations guides, workflows, runbooks, architecture diagrams, naming conventions, chart lifecycle documentation, security guidance, troubleshooting, responsibilities, and template-upgrade guidance. + +### Lifecycle Notes + +- Generated OCI DevOps resources are starter templates, not a continuously enforced CI/CD policy. +- Removing an application, component, tool, or manifest from configuration does not automatically delete its Git content or live Kubernetes resources. +- Production can target the same physical OKE cluster as pre-production for evaluation, but separate clusters are recommended for production isolation. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/README.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/README.md new file mode 100644 index 000000000..28095bab5 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/README.md @@ -0,0 +1,139 @@ +# OKE DevOps Starter + +[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oci-devops-rm-1.0.0/stack.zip) + +This OCI Resource Manager stack bootstraps OCI DevOps workflows for both application developers and OKE cluster administrators. + +It creates the starter structure for one or more applications. Each application has an umbrella chart for shared namespace resources, and each component has its own source repository, build pipeline, pull request pipeline, release flow, image repository, component chart, and deployment pipelines. + +The stack is intentionally a starting point, not a universal CI/CD policy. Customers select the initial applications, components, clusters, tools, naming, logging, notifications, and optional IAM through Resource Manager inputs. Developers and administrators then own application-specific tests, quality gates, chart values, deployment checks, and operational policies in the generated repositories and pipelines. + +For the shortest path from Resource Manager configuration to a working deployment, start with the **[Quickstart](docs/quickstart.md)**. + +The default sample creates: + +- DevOps project: `oke-devops-starter` +- Application: `sample-app` +- Components: `sample-api`, `sample-worker` +- Shared pipeline repository: `pipelines` +- Application chart repository: `sample-app-chart` +- Component source repositories: `sample-api`, `sample-worker` +- Optional cluster administration repository when enabled: `cluster-admin` + +## Documentation + +- [Quickstart](docs/quickstart.md): configure the Resource Manager form, apply the stack, deploy the first application, promote a release, and enable cluster operations. + +### Application Developers + +- [Developer Guide](docs/developers.md): where developers start and which repositories and pipelines they own. +- [Developer Runbooks](docs/developer-runbooks.md): add a component, customize tests, change code or charts, promote a release, and roll back. +- [Developer Workflow](docs/developer-workflow.md): the full SDLC from pull request to development, staging, approval, production, and final Git tagging. +- [Chart Lifecycle](docs/chart-lifecycle.md): the split between application baseline charts and independently deployed component charts. +- [Application Bootstrap](docs/application-bootstrap.md): parallel noprod/prod namespace and OCIR pull-secret initialization. + +### Cluster Operations + +- [Cluster Operations Guide](docs/cluster-operations.md): where OKE administrators start and how cluster configuration reaches noprod and prod. +- [Cluster Operations Runbooks](docs/cluster-operations-runbooks.md): add, configure, promote, and remove tools and Kubernetes resources. +- [Cluster Administration Reference](docs/cluster-administration.md): tool dependencies, chart mirroring, selective deployments, values artifacts, and supplemental resources. +- [Stack Operations](docs/operations.md): stack packaging, Resource Manager applies, validation, functional testing, and cleanup. + +### Shared Reference + +- [Changelog](CHANGELOG.md): release history and customer-visible capabilities. +- [Architecture](docs/architecture.md): what the stack creates and how the developer and operations paths fit together. +- [Responsibilities](docs/responsibilities.md): ownership boundaries for developers, cluster administrators, and stack/platform owners. +- [Resource Manager Inputs](docs/resource-manager-inputs.md): stack variables, application definitions, and optional cluster-administration configuration. +- [Naming Conventions](docs/naming-conventions.md): repository, pipeline, chart, image, namespace, artifact, and release naming rules. +- [Troubleshooting And Recovery](docs/troubleshooting-recovery.md): failure diagnosis, rollback behavior, partial failures, and drift. +- [Template Ownership And Upgrades](docs/template-ownership.md): release/development modes, repository seeding, and adopting newer templates. +- [Security Guidance](docs/security.md): credentials, IAM, networking, supply chain, and Kubernetes boundaries. +- [Glossary](docs/glossary.md): concise definitions of the solution's core terms. + +## Developer Quick Start + +```mermaid +flowchart LR + A["Configure Resource Manager stack"] --> B["Apply stack"] + B --> C["Run -bootstrap"] + C --> D["Deploy application baseline"] + D --> E["Open component pull request"] + E --> F["Merge to main"] + F --> G["Build SHA image and deploy dev"] + G --> H["Run release build with RC tag"] + H --> I["Deploy staging"] + I --> J["Approve production"] + J --> K["Deploy prod and tag final release"] +``` + +1. Select **Deploy to Oracle Cloud** at the top of this README. +2. Configure the OCI DevOps section, the OKE pre-prod environment, the OKE prod environment, the application list, and IAM options. Enable cluster administration only when this stack should also create the operations workflow. +3. Apply the stack. +4. Run `-bootstrap` with the OCIR username and Vault secret OCID. +5. Its independent noprod and prod stages initialize both namespaces and pull secrets in parallel. Run either stage alone when only one cluster should be initialized. +6. Publish the application baseline through `-package`; it triggers `-deploy` for noprod and approved prod delivery. +7. Develop through pull requests in each component repository. +8. Merge to `main` to build a 7-character SHA image and deploy the component to dev. +9. Run `-release-build` with a release candidate tag, such as `1.0.0-rc.1`, to promote through staging, approval, production, and final Git tagging. + +## Cluster Operations Quick Start + +Cluster administration is an optional path. Enable `enable_cluster_admin` only when this stack should also manage cluster tools and cluster-wide Kubernetes resources. + +```mermaid +flowchart LR + A["Enable cluster administration"] --> B["Apply stack"] + B --> C["Edit cluster-admin repository"] + C --> D["Open PR and validate"] + D --> E["Merge to main"] + E --> F["Detect changed cluster targets"] + F --> G["Mirror missing charts"] + G --> H["Orchestrate selected noprod changes"] + F --> I["Start cluster-admin-prod"] + I --> J["Approve production plan"] + J --> K["Orchestrate selected prod changes"] +``` + +1. Enable cluster administration and define the shared tool dependency graph in Resource Manager. +2. Apply the stack to create the `cluster-admin` repository and OCI DevOps pipelines. +3. Configure each public chart repository, chart name, and pinned version in the Resource Manager tool definition; the stack maintains `catalog/tools.yaml`. +4. Add cluster-specific values and supplemental resources under `clusters/noprod` and `clusters/prod`. +5. Open a pull request and merge it after `cluster-admin-pr` succeeds. +6. Let `cluster-admin-build` select only the affected tool or baseline stages. +7. Review and approve the production deployment before any prod mutation begins. + +Before execution, `cluster-admin-build` prints the selected clusters, dependency waves, actions, namespaces, chart versions, values versions, and final baseline work. Use `cluster-admin--decommission` for explicit tool removal; Git deletion alone never prunes live resources. + +## Application Configuration Example + +```json +[ + { + "name": "shop", + "components": [ + { "name": "invoice" } + ] + }, + { + "name": "orders", + "components": [ + { "name": "checkout" }, + { "name": "fulfillment" } + ] + } +] +``` + +Application names must be unique. Component names must be globally unique across all applications because each component gets a top-level OCI DevOps source repository and its own generated pipelines. + +## Design Principles + +- Keep developer inputs small and derive names from conventions. +- Build immutable images from `main` using only the 7-character Git SHA tag. +- Use pull requests for component validation before merging to `main`. +- Separate component delivery from application baseline delivery. +- Treat generated OCI DevOps resources as starter templates that teams can customize after creation. +- Seed repository files once and preserve developer-owned content on later stack applies. +- Keep production optional for early testing by allowing the prod OKE environment to point to the same cluster as pre-prod. +- Keep cluster administration configuration separate from application environments and deploy only changed cluster targets. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/checks.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/checks.tf new file mode 100644 index 000000000..7ca76c48d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/checks.tf @@ -0,0 +1,11 @@ +check "recommended_application_scale" { + assert { + condition = local.application_count <= 20 + error_message = "This starter stack is optimized for at most 20 applications in one OCI DevOps project. The apply will continue, but review OCI DevOps service limits and consider splitting projects." + } + + assert { + condition = local.component_count <= 50 + error_message = "This starter stack is optimized for at most 50 components in one OCI DevOps project. The apply will continue, but review repository, pipeline, and artifact service limits." + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/data.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/data.tf new file mode 100644 index 000000000..ee6f8526d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/data.tf @@ -0,0 +1,12 @@ +data "oci_identity_user" "current_user" { + user_id = var.current_user_ocid +} + +data "oci_identity_tenancy" "current_tenancy" { + tenancy_id = var.tenancy_ocid +} + +data "oci_identity_region_subscriptions" "region_subscriptions" { + tenancy_id = var.tenancy_ocid +} + diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/application-bootstrap.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/application-bootstrap.md new file mode 100644 index 000000000..c6e35963c --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/application-bootstrap.md @@ -0,0 +1,46 @@ +# Application Bootstrap + +`-bootstrap` is the manual, idempotent deployment pipeline used to prepare an application's namespaces and private-image pull secret. + +```mermaid +flowchart LR + Pipeline["-bootstrap"] --> Noprod["Initialize noprod namespace"] + Pipeline --> Prod["Initialize prod namespace"] + Noprod --> NoprodSecret["Create or replace OCIR pull secret"] + Prod --> ProdSecret["Create or replace OCIR pull secret"] +``` + +The noprod and prod shell stages both use the pipeline root as their predecessor, so a normal pipeline run executes them in parallel. OCI single-stage deployment can run either stage independently when only one cluster should be initialized or repaired. + +## Parameters + +Each application bootstrap pipeline has only three parameters: + +- `registry_username`: OCIR pull username, for example `/oracleidentitycloudservice/user@example.com`. +- `pull_password_secret_ocid`: OCI Vault secret OCID containing the OCIR pull password or auth token. +- `secret_name`: Kubernetes docker-registry secret name, default `ocirsecret`. + +Cluster OCIDs and namespaces are not user parameters. Terraform fixes them through stage tags and the configured noprod/prod OKE inputs, preventing a stage from initializing a namespace on the wrong cluster. + +## How It Works + +Each stage: + +1. Resolves its application, cluster, and namespace from OCI DevOps stage tags. +2. Selects the configured noprod or prod cluster OCID. +3. Creates kubeconfig using the OKE private endpoint. +4. Creates the namespace idempotently. +5. Reads the OCIR credential from OCI Vault. +6. Replaces only the configured docker-registry secret. + +The shell runner uses the worker subnet and optional worker NSG belonging to its target cluster. The component chart ServiceAccount references the pull secret explicitly; bootstrap does not modify the namespace default ServiceAccount. + +## Lifecycle Boundary + +Bootstrap does not package or deploy Helm charts. Application baseline changes belong exclusively to `-package` and `-deploy`, while component workloads use their own delivery pipelines. + +Run bootstrap after the stack creates a new application, when rotating the OCIR credential, or when repairing a missing namespace or pull secret. Repeated runs are safe. + +## Troubleshooting + +For `ImagePullBackOff`, verify that the relevant bootstrap stage succeeded, the Vault secret contains the actual OCIR token, the Kubernetes secret has type `kubernetes.io/dockerconfigjson`, and the component ServiceAccount references the configured secret name. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/architecture.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/architecture.md new file mode 100644 index 000000000..0d1ea7978 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/architecture.md @@ -0,0 +1,147 @@ +# Architecture + +The stack creates an OCI DevOps project that owns the operational path from code to OKE. It intentionally does not require a GitOps controller. + +Choose the path that matches your role: + +- [Application developers](developers.md) own component source, tests, images, component charts, and release promotion. +- [Cluster operations administrators](cluster-operations.md) own cluster tools, per-cluster values, supplemental resources, and cluster-wide objects. +- [Stack maintainers](operations.md) package, apply, validate, and upgrade the Resource Manager template. + +The [responsibility matrix](responsibilities.md) records the handoffs between these roles. + +```mermaid +flowchart TB + RM["OCI Resource Manager stack"] --> Project["OCI DevOps project"] + + Project --> PipelinesRepo["Repository: pipelines"] + Project --> AppChartRepo["Repository: -chart"] + Project --> ComponentRepo["Repository: "] + Project --> ClusterAdminRepo["Repository: cluster-admin"] + + Project --> BuildPipelines["Build pipelines"] + Project --> DeployPipelines["Deployment pipelines"] + Project --> Artifacts["Deploy artifacts"] + Project --> Triggers["Repository triggers"] + + AppChartRepo --> UmbrellaChart["Umbrella chart: "] + AppChartRepo --> ComponentChart["Component chart: /charts/"] + ComponentRepo --> SourceCode["Application source code"] + ComponentRepo --> PrSpec[".oci-devops/pull-request-pipeline.yaml"] + + BuildPipelines --> ComponentBuild["-build"] + BuildPipelines --> ComponentPr["-pr"] + BuildPipelines --> ReleaseBuild["-release-build"] + BuildPipelines --> AppPackage["-package"] + + DeployPipelines --> AppBootstrap["-bootstrap"] + DeployPipelines --> AppDeploy["-deploy"] + DeployPipelines --> DevDeploy["-dev-deploy"] + DeployPipelines --> ReleaseDeploy["-release"] + + ComponentBuild --> OCIRImage["OCIR image"] + ComponentBuild --> OCIRComponentChart["OCIR component chart"] + AppPackage --> OCIRAppChart["OCIR umbrella chart"] + + AppDeploy --> OkeNoprod["OKE pre-prod"] + DevDeploy --> OkeNoprod + ReleaseDeploy --> OkeNoprod + ReleaseDeploy --> OkeProd["OKE prod"] + AppBootstrap --> OkeNoprod + AppBootstrap --> OkeProd + ClusterAdminRepo --> AdminBuild["cluster-admin-build"] + AdminBuild --> ClusterNoprod["cluster-admin-noprod\norchestrator"] + AdminBuild --> ClusterProd["cluster-admin-prod\napproval + orchestrator"] + ClusterNoprod --> OkeNoprod + ClusterProd --> OkeProd +``` + +## Main Resources + +For the whole stack: + +- One OCI DevOps project. +- One shared `pipelines` repository containing common build specs and scripts. +- One notification topic, unless an existing topic is supplied. +- Optional IAM policies and dynamic group for DevOps access. +- One pre-prod OKE deploy environment named from the selected cluster. +- One prod OKE deploy environment named from the selected prod cluster. +- One `cluster-admin` repository, one immutable Generic Artifact repository for tool values, and shared PR/build/chart-mirror pipelines. +- One orchestration pipeline per cluster; prod adds a mandatory approval before its orchestrator. +- One explicit tool decommission pipeline per OKE target, with mandatory approval in prod. + +For each application: + +- One idempotent `-bootstrap` pipeline with parallel noprod/prod namespace initialization stages. +- One application chart repository, by default `-chart`. +- One umbrella chart under ``. +- One application baseline package pipeline: `-package`. +- One application baseline deployment pipeline: `-deploy`. +- Noprod and prod values artifacts for the application baseline chart. + +For each component: + +- One source repository: ``. +- One component chart under `/charts/`. +- One build pipeline: `-build`. +- One pull request pipeline: `-pr`. +- One release build pipeline: `-release-build`. +- One dev deployment pipeline: `-dev-deploy`. +- One release deployment pipeline: `-release`. +- Dev, staging, and prod values artifacts for the component chart. + +The project also has two shared component command specs for release image promotion and final commit tagging. Component pipelines provide generic image and repository parameters, so these command specs do not multiply with the number of components. + +## Repository Roles + +```mermaid +flowchart LR + subgraph Shared["Shared"] + P["pipelines repo\ncommon build specs and scripts"] + end + + subgraph Application["Per application"] + A["-chart repo"] + U["umbrella chart\nshared namespace baseline"] + C["component charts\n/charts/"] + A --> U + A --> C + end + + subgraph Component["Per component"] + S[" source repo"] + PR["PR build spec\nowned by developers"] + App["application code"] + S --> PR + S --> App + end + + P --> A + P --> S +``` + +`pipelines` contains reusable delivery logic. It owns the common build specs and shell scripts used by generated build pipelines. + +`-chart` contains the umbrella chart and nested component charts. The umbrella chart is for shared namespace resources. Component charts are deployed independently. + +`` contains the application source code. Its `.oci-devops/pull-request-pipeline.yaml` is intentionally component-owned because tests depend on the component's language and framework. + +## Template Ownership + +Terraform creates and wires the DevOps resources, but the generated pipelines are starter templates. Application/component DevOps resources use Terraform lifecycle ignore rules so a team can modify names, descriptions, build specs, parameters, tags, and stage internals after bootstrap without every apply forcing them back. + +Repository content follows a create-only ownership model: + +| Seed unit | Behavior after creation | +| --- | --- | +| Shared pipeline scripts and generic files | Seeded only when the `pipelines` repository is empty | +| Application/component pipeline specs | Missing entity-specific files are added; existing files are preserved | +| Component source repository | Seeded only when the repository is empty | +| Application baseline chart | Seeded only when the chart repository is empty | +| Component chart | A missing component directory is added; an existing directory is preserved | + +Template changes in a newer stack archive are not pushed into repositories that developers already own. Teams can adopt those changes explicitly through their normal Git review process. + +IAM, networking inputs, OCIR conventions, and core stack structure remain Terraform-managed. + +Cluster administration uses the same template ownership model. The repository is seeded only while empty, and generated pipelines, stages, artifacts, triggers, and branch protection preserve later administrator customization. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/chart-lifecycle.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/chart-lifecycle.md new file mode 100644 index 000000000..f0d48af34 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/chart-lifecycle.md @@ -0,0 +1,70 @@ +# Chart Lifecycle + +The solution separates the application baseline chart from component charts. + +```mermaid +flowchart TB + ChartRepo["-chart repository"] + ChartRepo --> Umbrella["\numbrella baseline chart"] + ChartRepo --> Components["/charts/\ncomponent chart"] + + Umbrella --> AppPackage["-package"] + AppPackage --> AppDeploy["-deploy"] + AppDeploy --> BaselineNoprod["Release -noprod\npre-prod cluster"] + AppDeploy --> BaselineProd["Release \nprod cluster"] + + Components --> ComponentBuild["-build"] + ComponentBuild --> ComponentDev["-dev-deploy"] + ComponentDev --> DevRelease["Release -dev\napplication namespace"] +``` + +## Application Baseline Chart + +The umbrella chart is named after the application, for example `sample-app` or `shop`. + +It owns namespace-wide resources that are common to all components in the application. It has no component dependencies and should not be used to deploy component workloads in this starter flow. + +The chart repository keeps standalone component charts under `/charts`, but the umbrella chart's `.helmignore` excludes that directory when the baseline is loaded or packaged. Adding a component therefore does not require changing the umbrella `Chart.yaml` or `values.yaml`. + +When umbrella chart files change under `/**`, excluding `/charts/**`: + +1. OCI DevOps triggers `-package`. +2. The pipeline packages only the umbrella chart. +3. The pipeline triggers `-deploy`. +4. The baseline deploy installs noprod release `-noprod`. +5. After approval, it installs prod release ``. + +Noprod and prod baseline values are cluster-specific. They are not dev/staging/prod component values. + +## Component Charts + +```mermaid +flowchart LR + A["Component chart commit"] --> B["-build"] + B --> C["Skip image build"] + C --> D["Package component chart only"] + D --> E["Push chart to OCIR"] + E --> F["Redeploy dev"] +``` + +Each component chart lives under the application chart repository: + +```text +/charts/ +``` + +The component chart is published as its own OCI Helm chart and deployed directly by component deployment pipelines. + +When component chart files change: + +1. OCI DevOps triggers `-build`. +2. The pipeline packages only that component chart. +3. The dev component release is redeployed. + +The application umbrella chart version and component chart versions are independent. A component chart change should not require editing the parent `Chart.yaml`. + +## Why Split Them + +The umbrella chart gives the application one place for shared namespace resources. Component deploys stay fast and focused because they do not reinstall every component in the application. + +This also lets dev and staging releases for the same component coexist in the same namespace, while prod can use a separate cluster and a cleaner release name. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-administration.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-administration.md new file mode 100644 index 000000000..ffda13172 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-administration.md @@ -0,0 +1,142 @@ +# Cluster Administration + +The cluster-administration workflow is independent of application promotion. Configuration is organized by physical OKE cluster, with one OCI DevOps deployment pipeline for `noprod` and one for `prod`. + +Enable the workflow with the Resource Manager `enable_cluster_admin` toggle. It defaults to disabled so developer-only stacks do not create operational repositories, artifacts, or pipelines. Disabling it later removes the OCI resources managed by this feature but does not uninstall deployed Helm releases or delete live Kubernetes objects. + +```mermaid +flowchart LR + PR["Cluster admin pull request"] --> Validate["cluster-admin-pr"] + Validate --> Merge["Merge to main"] + Merge --> Build["cluster-admin-build"] + Build --> Detect["Detect changed stages"] + Detect --> Mirror["Mirror missing charts"] + Detect --> Target{"Target cluster"} + Target -->|noprod| Wave1["Dependency wave 1"] + Target -->|prod| Approval["Approve production changes"] + Approval --> Wave1 + Wave1 --> Wave2["Dependency wave 2"] + Wave2 --> Baseline["Cluster-wide resources"] + Baseline --> Done["Selected stages complete"] +``` + +## Repository Layout + +```text +catalog/ + tools.yaml +clusters/ + noprod/ + baseline/ + tools/ + cert-manager/ + tool.yaml + values.yaml + resources/ + traefik/ + tool.yaml + values.yaml + resources/ + verify.sh + prod/ + baseline/ + tools/ +``` + +`catalog/tools.yaml` is generated from the Resource Manager tool definitions and pins public chart repositories and versions. Change chart coordinates through the stack input rather than editing this generated file. The mirror pipeline copies missing versions into: + +```text +/charts/cluster-tools/ +``` + +The target path uses the upstream chart name because Helm derives the OCI repository name from chart metadata. The logical tool name remains the pipeline, stage, namespace-default, and tagging identity. + +The mirror pipeline removes only non-ASCII chart annotations before pushing because OCIR normalizes that OCI manifest metadata. Chart templates, values, dependencies, and runtime behavior are unchanged. + +Chart sources may be traditional `https://` Helm repositories or `oci://` repository base paths. HTTPS sources use `helm repo add`; OCI sources are pulled directly from `/`. Public OCI sources require no extra credentials. Private sources require credentials already available to Helm; the standard pipeline authenticates only to the target OCIR registry. + +## Tool Dependencies + +Each `tool.yaml` declares its namespace and direct prerequisites: + +```yaml +name: traefik +namespace: traefik +depends_on: + - cert-manager +``` + +The same dependency must be present in the shared Resource Manager `cluster_administration.tools` input because both cluster paths use the same topology. Validation rejects unknown dependencies, self-dependencies, duplicates, and cycles. + +Independent changed tools run in parallel. A selected tool waits for selected prerequisites, and changing a prerequisite also selects its downstream dependents. + +## Deployment Pipelines + +Both cluster pipelines use the same orchestration pattern: + +- `cluster-admin-noprod` contains one root shell orchestrator. +- `cluster-admin-prod` contains one mandatory manual approval followed by the same shell orchestrator. + +The build publishes each validated cluster deployment plan as an immutable Generic Artifact versioned by the exact Git commit. The orchestrator downloads that plan and commit, validates the complete repository structure, downloads immutable values artifacts, performs atomic Helm upgrades in dependency waves, applies supplemental resources, runs optional verification scripts, and applies selected baseline resources last. + +Both orchestration stages reuse `cluster-admin-deploy-command-spec`. Explicit removal pipelines reuse `cluster-admin-decommission-command-spec`. + +Each cluster pipeline exposes only three stable parameters: `config_commit`, `cluster_id`, and `cluster_name`. No per-tool parameters or OCI stages are required because each orchestrator derives its selected plan from `config_commit`. + +Dependency ordering is enforced inside each orchestrator. Independent tools run in parallel topological waves, and each tool's Helm action precedes its supplemental resources. + +Merge through the repository so `cluster-admin-build` computes and publishes the exact change set. A manual cluster run must provide a full `config_commit` whose immutable deployment-plan artifact exists. + +Before any chart mirror, approval, or mutation, the build prints a deterministic execution plan. It lists the exact configuration commit, selected cluster, dependency waves, Helm/resource actions, namespaces, chart versions, commit-versioned values, and final baseline work. The `cluster-admin-prod` deployment carries a compact target summary for the approver. + +## Change Selection + +| Changed path | Selected action | +| --- | --- | +| `clusters//baseline/*.yaml` | Baseline stage | +| `clusters//tools//values.yaml` | Tool Helm stage | +| `clusters//tools//tool.yaml` | Tool Helm stage | +| `clusters//tools//resources/*.yaml` | Supplemental resources stage | +| `clusters//tools//verify.sh` | Supplemental resources stage | +| `catalog/tools.yaml` | Helm stages for configured tools | + +When both parts of a tool change, Helm runs first and supplemental resources follow. Independent tools run in dependency waves. If cluster-wide resources are selected in the same change, the baseline runs only after all selected tool work completes. Prod pauses inside `cluster-admin-prod` before its orchestrator mutates the cluster; noprod begins immediately. + +## Baseline Resources + +Place ClusterRoles, ClusterRoleBindings, StorageClasses, custom resources, and other non-namespaced objects not owned by a tool chart under `clusters//baseline`. + +Tool Helm stages install chart-bundled CRDs. The baseline stage then downloads the exact Git commit, performs a server-side dry run, rejects objects resolved to a namespace, and applies them with field manager `oci-devops-cluster-admin`. This ordering allows baseline custom resources to use APIs introduced by the tool charts. + +## Tool Deployment + +Both orchestrators use `helm upgrade --install --atomic`, create the namespace when needed, consume the immutable values artifact, apply `resources/*.yaml` afterward, and run `verify.sh` when present. + +Supplemental resources must resolve to the configured namespace. Plain Kubernetes `Secret` objects and cross-namespace resources are rejected. Use External Secrets Operator resources when a tool needs secret material. + +## Resource Tags + +Cluster-administration resources use a common freeform-tag contract: + +| Tag | Meaning | +| --- | --- | +| `owner=cluster-administrators` | Operational ownership | +| `purpose=cluster-administration` | Stable resource discovery key used by the dispatcher | +| `scope=operations` | Separates operations resources from developer delivery resources | +| `cluster=` | Physical target cluster, when applicable | +| `tool=` | Kubernetes tool, when applicable | +| `role=` | Resource function such as `helm`, `resources`, `baseline`, or `chart-mirror` | + +The tags are applied to Terraform-created repositories, artifact repositories, deployment pipelines, stages, artifacts, triggers, and executable build stages. Runtime-created mirror runs and cluster pipeline deployments are tagged too. New OCIR chart repositories receive the same ownership tags when the mirror pipeline creates them. + +The top-level cluster-admin build pipeline containers are the one exception: the current OCI Terraform provider cannot safely update tags on parameterless build pipelines. Their executable build stages carry the complete operations tag set instead. + +## Lifecycle Contract + +- Values are immutable Generic Artifacts versioned by the full configuration commit that selected the Helm action. +- Both orchestrators log the exact commit, chart version, values artifact version, and Helm release history; rollback uses `helm rollback` rather than OCI native Helm-stage rollback. +- Removing a manifest from Git does not delete the live object. +- Out-of-band drift is not continuously reconciled. +- Deletion remains an explicit administrator operation. +- `cluster-admin--decommission` deletes repository-declared supplemental resources and uninstalls the selected Helm release. Prod requires approval and the namespace is retained. +- Terraform creates both cluster orchestrators, then preserves later OCI customization through lifecycle ignores. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations-runbooks.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations-runbooks.md new file mode 100644 index 000000000..1f243a01c --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations-runbooks.md @@ -0,0 +1,84 @@ +# Cluster Operations Runbooks + +These runbooks assume `enable_cluster_admin=true` and an initialized `cluster-admin` repository. + +## Add A Tool + +Adding a tool changes the administrator-owned repository and the topology consumed by both orchestrators; it does not create per-tool OCI stages. + +1. Add the globally unique tool name, public Helm repository, upstream chart name, pinned version, namespace, and direct `depends_on` entries to the Resource Manager `cluster_administration.tools` list. +2. Apply the stack to create the tool stages in both `cluster-admin-noprod` and `cluster-admin-prod`. +3. Confirm the stack added the configured chart source to `catalog/tools.yaml`. +4. Review the seeded `tool.yaml` and `values.yaml` under both `clusters/noprod/tools/` and `clusters/prod/tools/`. +5. Configure noprod first. Keep prod values valid, but deploy the prod configuration only through a reviewed prod change and approval. +6. Add optional `resources/*.yaml` and `verify.sh` files. +7. Open a pull request and make `cluster-admin-pr` pass. +8. Merge to `main` and confirm the missing chart is mirrored before the selected deployment stages run. + +The shared topology creates the tool stages for both clusters. The namespace defaults to the tool name; keep explicit namespaces synchronized between Resource Manager and both generated `tool.yaml` files. + +## Define Tool Dependencies + +1. Put only direct prerequisites in `depends_on`; transitive dependencies are derived. +2. Declare the same dependency topology in Resource Manager and each tool's `tool.yaml`. +3. Open a pull request and let validation reject unknown tools, self-dependencies, duplicates, or cycles. +4. After merge, confirm independent tools run in parallel and dependent tools start in later waves. + +When a prerequisite changes, the dispatcher also selects downstream dependents. + +## Change Cluster Values + +1. Edit only `clusters//tools//values.yaml`. +2. Open and merge a pull request. +3. Confirm `cluster-admin-build` publishes an immutable Generic Artifact with the full Git commit SHA as its version. +4. Confirm only the selected cluster/tool Helm stage runs, plus any required downstream dependents. +5. For prod, approve the production approval deployment before mutation begins. +6. Verify the Helm release in the tool namespace. + +Noprod and prod values are independent even though the tool topology is shared. + +## Add Supplemental Namespaced Resources + +1. Add YAML under `clusters//tools//resources`. +2. Set the resource namespace to the configured tool namespace, or omit it when the resource type and script safely default it. +3. Do not commit plain Kubernetes `Secret` objects. Use External Secrets or another reference to an external secret store. +4. Open and merge a pull request. +5. Confirm the tool Helm stage runs before the supplemental-resource stage when both are selected. + +Validation rejects explicit cross-namespace resources. + +## Add Cluster-Wide Resources + +1. Add non-namespaced YAML under `clusters//baseline`. +2. Use this location for resources such as ClusterRoles, ClusterRoleBindings, StorageClasses, and custom resources that are not owned by a tool chart. +3. Open and merge a pull request. +4. Confirm the baseline stage validates that every object is cluster-scoped. +5. Confirm selected tool waves finish before the baseline stage, allowing baseline objects to use tool-installed CRDs. + +## Promote A Tool Configuration To Prod + +There is no automatic environment promotion. Prod is a separate physical-cluster configuration. + +1. Copy or adapt the reviewed noprod values and resources into `clusters/prod/tools/`. +2. Review cluster-specific endpoints, storage classes, load balancer annotations, identities, and capacity. +3. Open and merge a pull request. +4. Confirm `cluster-admin-prod` starts and pauses at its approval stage. +5. Review the target summary and approve it. +6. Verify the production orchestrator deploys only the selected targets and applies baseline resources last. + +When noprod and prod point to the same test cluster, avoid conflicting release names, namespaces, cluster-scoped objects, and external endpoints. + +## Remove A Tool + +Removal is explicit because deleting Git files does not prune live resources. + +1. Keep the tool configuration on `main` so the removal pipeline can validate its supplemental resources. +2. Identify cluster-scoped objects not owned by the Helm release; the pipeline intentionally does not infer or delete those objects. +3. Run `cluster-admin--decommission` with `tool_name` and `tool_namespace`. +4. Approve the deployment when removing from prod. +5. Confirm supplemental resources are absent and the Helm release is uninstalled. The namespace is deliberately retained. +6. Verify the namespace contains no unrelated workloads before deleting it manually. +7. Remove dependent references, then remove the tool from Git and the Resource Manager topology. +8. Apply the stack only after live cleanup and dependency review are complete. + +Neither cluster has per-tool Helm stages. Use the explicit decommission pipeline for removal. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations.md new file mode 100644 index 000000000..ecaab9fcd --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/cluster-operations.md @@ -0,0 +1,69 @@ +# Cluster Operations Guide + +This section is for OKE cluster administrators. It covers shared cluster tools, cluster-scoped Kubernetes objects, and supplemental namespaced resources. It is separate from the component SDLC: there are no dev, staging, or release-candidate promotions in the cluster administration path. + +Enable this workflow with `enable_cluster_admin`. When disabled, the stack creates only the developer delivery resources. + +```mermaid +flowchart TB + Config["cluster-admin repository"] --> PR["PR validation"] + PR --> Merge["Merge to main"] + Merge --> Detect["Detect changed cluster and tools"] + Detect --> Mirror["Mirror missing pinned charts"] + Mirror --> Target{"Target cluster"} + Target --> Noprod["cluster-admin-noprod orchestrator"] + Target --> Prod["cluster-admin-prod"] + Prod --> ProdApproval["Production approval"] + ProdApproval --> ProdDeploy["Approved production orchestrator"] + Noprod --> NoprodBaseline["Selected noprod baseline last"] + ProdDeploy --> ProdBaseline["Selected prod baseline last"] +``` + +## Administrator-Owned Repository + +The `cluster-admin` repository is the source of truth for operations configuration: + +```text +catalog/tools.yaml +clusters/noprod/baseline/ +clusters/noprod/tools// +clusters/prod/baseline/ +clusters/prod/tools// +``` + +- `catalog/tools.yaml` is generated from the Resource Manager input and pins each public Helm repository, chart, and version. +- `values.yaml` contains cluster-specific chart configuration. +- `resources/*.yaml` contains supplemental resources for the tool namespace. +- `baseline/*.yaml` contains cluster-scoped resources not owned by a tool chart. +- `tool.yaml` declares the namespace and dependencies on other tools. + +## Delivery Model + +1. Define the shared tool topology and pinned public chart sources in the Resource Manager `cluster_administration` input. +2. Maintain per-cluster values and supplemental resources in the repository. +3. Open a pull request. `cluster-admin-pr` validates structure, YAML, namespaces, catalog references, and dependencies without changing a cluster. +4. Merge to `main`. `cluster-admin-build` identifies changed targets and mirrors missing chart versions into OCIR. +5. Both clusters start a full pipeline deployment whose orchestrator consumes the immutable selected-target plan and runs independent tools in dependency waves. +6. Noprod starts immediately; prod pauses at its approval stage before invoking the same orchestrator. +7. Both paths deploy Helm charts before supplemental resources and apply selected cluster-wide baseline resources last. + +Values artifacts and cluster deployment plans are immutable and use the full configuration commit SHA as their version. Each orchestrator logs exact chart and values versions; Helm keeps release history for explicit rollback. + +## Operational Boundaries + +- The workflow applies desired changes but does not continuously reconcile drift. +- Removing a manifest from Git does not prune the live object. +- Supplemental resources cannot target a namespace other than the configured tool namespace. +- Secret material must not be committed; use resources such as External Secrets that refer to an external secret store. +- If noprod and prod point to the same physical cluster during testing, use distinct tool namespaces or avoid deploying both configurations concurrently when the charts would conflict. + +## Continue Reading + +- [Cluster Operations Runbooks](cluster-operations-runbooks.md) gives step-by-step procedures for adding, configuring, promoting, and removing tools. +- [Cluster Administration Reference](cluster-administration.md) documents the repository contract, DAG behavior, selective stage execution, tags, and lifecycle rules. +- [Responsibilities](responsibilities.md) clarifies the boundary between cluster administrators, developers, and stack owners. +- [Troubleshooting And Recovery](troubleshooting-recovery.md) covers partial DAG failures, approval behavior, rollback, and drift. +- [Security Guidance](security.md) describes secret handling, IAM, pinned charts, and cluster-level review boundaries. +- [Resource Manager Inputs](resource-manager-inputs.md) explains the feature flag and tool topology JSON. +- [Stack Operations](operations.md) covers stack packaging, applies, validation, functional tests, and cleanup. +- [Architecture](architecture.md) shows the operations path alongside application delivery. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-runbooks.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-runbooks.md new file mode 100644 index 000000000..805a707d6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-runbooks.md @@ -0,0 +1,78 @@ +# Developer Runbooks + +These runbooks cover routine component work. Replace placeholders such as `` and `` with the names derived from the Resource Manager application configuration. + +## Add A Component + +Adding a component changes the stack topology and therefore requires the stack/platform owner. + +1. Add a globally unique component name to the application's `components` list in Resource Manager. +2. Apply the stack. Terraform creates the missing repository, pipelines, artifacts, triggers, and chart path without replacing existing repository files. +3. Clone the new `` repository and replace the seeded source with the component implementation. +4. Replace the placeholder PR build specification with language-specific tests. +5. Implement a multi-stage `Dockerfile` that produces the runtime image. +6. Review the seeded chart under `-chart//charts/`. +7. Open pull requests for source and chart changes. Do not commit directly to protected `main` branches. + +Expected result: merging source to `main` builds `:` and deploys Helm release `-dev`. + +Before the first component deployment, run `-bootstrap` with the OCIR username and Vault secret OCID. A full run initializes noprod and prod in parallel; use an OCI single-stage deployment for only one target. Publish and deploy the application baseline separately through `-deploy`. + +## Customize PR Validation + +Edit `.oci-devops/pull-request-pipeline.yaml` in the component repository. + +1. Keep tests independent of image publishing and deployment. +2. Install or invoke language-specific tooling inside the build specification. +3. Run unit tests first, then integration, contract, lint, or security checks as appropriate. +4. Make every failed check return a nonzero exit code. +5. Open or update a pull request and confirm `-pr` is triggered. + +The seeded `Unit tests succeeded` command is only a placeholder and is not a real quality gate. + +## Change Component Runtime Code + +1. Create a feature branch in ``. +2. Commit the source and `Dockerfile` changes. +3. Open a pull request to `main` and make `-pr` pass. +4. Merge using the configured repository strategy. +5. Confirm `-build` creates only the 7-character SHA image tag. +6. Confirm `-dev-deploy` updates release `-dev` in the application namespace. + +Do not manually create a version image tag during this workflow. Release tags are created by promotion. + +## Change A Component Chart + +1. Create a branch in `-chart`. +2. Change only `/charts//**`. +3. Increment the component chart version in its `Chart.yaml`. +4. Open and merge a pull request. +5. Confirm only `-build` runs for this chart path. +6. Confirm it skips the image build, publishes the component chart, and redeploys `-dev` with the latest source SHA image. + +Do not increment the umbrella chart version for a component-only change. + +## Promote A Release Candidate + +1. Verify the desired commit has a successful main build and a corresponding `` image. +2. Run `-release-build` with `release_tag`, for example `1.2.0-rc.1`. +3. Optionally provide the full `commit_id`; otherwise the pipeline resolves current `main`. +4. Confirm the RC Git tag and image tag point to the selected commit and image digest. +5. Verify `-staging` in the pre-prod cluster. +6. Review the successful staging deployment and approve production. +7. Confirm the production-only Helm status stage succeeds. +8. Confirm release ``, final image `1.2.0`, and final Git tag `1.2.0` exist after production succeeds. + +Never reuse an RC or final version for a different commit. Choose a new RC number or release version. + +## Roll Back A Component + +OCI DevOps retains the chart and values snapshot used by a deployment, even if the current artifact definition later changes. + +1. Identify the last successful deployment with the desired chart, image tag, and values. +2. Use OCI DevOps rollback for the affected component deployment pipeline. +3. Verify the Helm release revision and workload image in the target namespace. +4. Confirm readiness and component health before closing the incident. +5. Fix forward through a new commit and RC; do not move an existing immutable image or Git tag. + +A production rollback does not rewrite release history or delete the Git and image tags created by the failed/newer release. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-workflow.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-workflow.md new file mode 100644 index 000000000..d493096db --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developer-workflow.md @@ -0,0 +1,153 @@ +# Developer Workflow + +This is the intended SDLC for a component. + +```mermaid +sequenceDiagram + participant Dev as Developer + participant Repo as Component Repo + participant PR as Component PR Pipeline + participant Build as Component Build Pipeline + participant DevDeploy as Component Dev Deploy + participant ReleaseBuild as Component Release Build + participant Release as Component Release Pipeline + participant OKE as OKE + + Dev->>Repo: Push feature branch + Dev->>Repo: Open PR to main + Repo->>PR: Trigger PR validation + PR-->>Dev: Unit/integration checks + Dev->>Repo: Merge PR + Repo->>Build: Trigger main build + Build->>Build: Build multi-arch image tagged sha7 + Build->>DevDeploy: Trigger dev deploy + DevDeploy->>OKE: Deploy -dev + Dev->>ReleaseBuild: Run with release_tag 1.0.0-rc.1 + ReleaseBuild->>ReleaseBuild: Tag commit and retag image to RC + ReleaseBuild->>Release: Trigger release deployment + Release->>OKE: Deploy staging + Release-->>Dev: Wait for approval + Dev->>Release: Approve production + Release->>Release: Retag RC image to final version + Release->>OKE: Deploy prod + Release->>Repo: Tag final release after prod succeeds +``` + +## Pull Request + +1. Create a branch in the component repository. +2. Open a pull request targeting `main`. +3. OCI DevOps triggers `-pr`. +4. The PR pipeline runs `.oci-devops/pull-request-pipeline.yaml` from the component repository. + +The seeded PR pipeline is a placeholder that prints `Unit tests succeeded`. Developers should replace it with real unit and integration tests for the component. + +This is intentionally empty in the template because a useful test strategy depends on the component language, framework, runtime, and integration boundaries. For example, a Java service may run Maven or Gradle tests, a Node.js service may run npm scripts, and a worker component may need contract or queue integration tests. + +## Main Build And Dev Deployment + +```mermaid +flowchart LR + A["Merge to main"] --> B["-build"] + B --> C["Build multi-arch image"] + C --> D["Push image:"] + B --> E["Ensure component chart exists"] + E --> F["Package chart if missing"] + D --> G["Export image repository and sha7 tag"] + F --> G + G --> H["-dev-deploy"] + H --> I["Helm release -dev"] +``` + +After the pull request is merged to `main`: + +1. OCI DevOps triggers `-build`. +2. The build pipeline builds a multi-architecture image. +3. The image is pushed with only the 7-character commit SHA tag. +4. The pipeline ensures the component chart version exists in OCIR, packaging it when needed. +5. The pipeline triggers `-dev-deploy`. +6. Dev deploy installs or upgrades Helm release `-dev`. + +Normal main builds do not create version tags such as `1.0.0`. Version tags are created only by release flows. + +The shared build pipeline expects the component repository to describe how the runtime artifact is built through its `Dockerfile`. Use a multi-stage Dockerfile when the component needs compilation, dependency installation, asset bundling, or test/build tooling that should not be present in the final runtime image. This keeps the OCI DevOps build pipeline generic while letting each component own its language-specific build details. + +## Component Chart Change + +When files under `/charts//**` change: + +1. OCI DevOps triggers `-build`. +2. The pipeline skips the image build. +3. The component chart is packaged and pushed. +4. Dev is redeployed using the latest source SHA image tag. + +This lets chart-only changes be tested in dev without rebuilding the application image. + +## Release Candidate + +```mermaid +flowchart TB + A["Manual release build"] --> B["Input release_tag\nexample 1.0.0-rc.1"] + A --> C["Optional commit_id"] + C --> D["Resolve source commit"] + D --> E["Compute sha7 image tag"] + E --> F["Verify source image exists"] + F --> G["Create OCI DevOps Git RC tag"] + G --> H["skopeo copy\nsha7 image to RC image"] + H --> I["Verify matching digests"] + I --> J["Trigger -release"] +``` + +Run `-release-build` manually with: + +- `release_tag`: strict SemVer RC, for example `1.0.0-rc.1`. +- `commit_id`: optional full commit SHA. If omitted, the current `main` commit is used. + +The release build: + +1. Resolves the source commit. +2. Computes the source image tag from the first 7 characters of the commit SHA. +3. Verifies that image exists. +4. Creates the OCI DevOps Git tag for the RC. +5. Retags the SHA image to the RC tag using skopeo in a Docker container. +6. Verifies source and target image digests match. +7. Triggers `-release`. + +## Staging, Approval, And Production + +```mermaid +flowchart LR + A["-release"] --> B["Deploy staging\nrelease -staging"] + B --> D["Manual approval"] + D --> E["Retag image\n1.0.0-rc.1 to 1.0.0"] + E --> F["Deploy prod\nrelease "] + F --> G["Tag source commit\n1.0.0"] + G --> H["Report prod Helm status"] +``` + +`-release` performs the production promotion path: + +1. Deploy the RC image to staging as release `-staging`. +2. Review the successful staging deployment and approve or reject production. +3. Retag the approved RC image as the final release image, for example `1.0.0-rc.1` to `1.0.0`. +4. Deploy the final image to prod as release ``. +5. Tag the source commit with the final Git tag, for example `1.0.0`. +6. Report production Helm status, resources, revision history, notes, and the namespace-scoped release listing. + +The final tag happens after production deployment succeeds. If the final tag already exists on the same commit, the tag stage treats it as an idempotent success. If it points to a different commit, it fails. The final Helm status stage is informational and runs only after deployment and tagging are complete. + +Component build and deployment pipelines use one generic parameter contract: + +- `component_chart_version` +- `image_repository` +- `image_tag` + +The release pipeline interprets `image_tag` as the release candidate tag. Approval follows the successful staging deployment. The promotion shell stage exports `release_image_tag` for subsequent shell logic and diagnostics, but OCI does not substitute command-spec exports into a later Helm values artifact. Production therefore retains the RC `image_tag`; after promotion succeeds, the component chart strips the terminal `-rc.N` when `environment: prod`. Repository identity is a Terraform-derived default on the component release pipeline. The image-promotion and final Git-tag command specs are shared by every component. + +## Versioning Rules + +- Main image tag: exactly 7 Git SHA characters, such as `a1b2c3d`. +- Release candidate tag: strict SemVer RC, such as `1.0.0-rc.1`. +- Final release tag: strict SemVer, such as `1.0.0`. + +The final image tag and final Git tag are derived from the release candidate by removing `-rc.N`. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developers.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developers.md new file mode 100644 index 000000000..ff18b4ee3 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/developers.md @@ -0,0 +1,55 @@ +# Developer Guide + +This section is for application and component developers. The stack gives each component an independent source repository, validation pipeline, image build, Helm chart, dev deployment, and release path. Developers do not need to maintain the cluster tool catalog or cluster-wide Kubernetes resources. + +```mermaid +flowchart LR + Branch["Feature branch"] --> PR["Pull request"] + PR --> Tests["-pr"] + Tests --> Main["Merge to main"] + Main --> Build["-build"] + Build --> Image["Multi-arch image:"] + Image --> Dev["Deploy dev"] + Dev --> RC["Create release candidate"] + RC --> Staging["Deploy staging"] + Staging --> Approval["Review diff and approve"] + Approval --> Prod["Deploy production"] + Prod --> Final["Final image and Git tag"] +``` + +## Repositories Developers Use + +| Repository | Ownership and purpose | +| --- | --- | +| `` | Application code, `Dockerfile`, and `.oci-devops/pull-request-pipeline.yaml` | +| `-chart` | Component charts under `/charts/` | +| `pipelines` | Platform-owned starter build specifications and scripts; change only when evolving the common delivery model | + +The seeded PR build specification is intentionally a placeholder. Replace it with tests appropriate to the component's language, runtime, and integration boundaries. The generic main build expects a `Dockerfile`; use a multi-stage Dockerfile to keep language-specific compilation and tooling inside the component repository. + +The default main build specification is `-build-pipeline.yaml`. Stack configuration can set a component `build_spec_path` in the `pipelines` repository. Several components can share a specification such as `java/java-build-pipeline.yaml`. Resource Manager creates a missing custom path and its parent folders from the default template once; the DevOps engineer owns all later changes and Resource Manager never refreshes the file. + +## Daily Workflow + +1. Create a feature branch in the component repository. +2. Open a pull request to `main` and make `-pr` pass. +3. Merge the pull request. `-build` publishes a multi-architecture image tagged with the 7-character Git SHA and deploys it to dev. +4. Test component chart changes in dev by changing only that component's chart directory. +5. When the commit is ready for promotion, run `-release-build` with a tag such as `1.0.0-rc.1`. +6. Verify staging and approve production. +7. Confirm the final image and source commit are tagged `1.0.0` after production succeeds. + +## Platform Prerequisites + +Before a component can deploy, a platform owner runs `-bootstrap`. Its independent noprod and prod stages initialize the configured namespaces and OCIR pull secrets in parallel. The application baseline remains owned exclusively by `-deploy`. Component developers normally consume those resources rather than manage them. + +## Continue Reading + +- [Developer Runbooks](developer-runbooks.md) gives step-by-step procedures for routine component work and rollback. +- [Developer Workflow](developer-workflow.md) describes every pipeline stage and versioning rule. +- [Chart Lifecycle](chart-lifecycle.md) explains application baseline and component chart ownership. +- [Responsibilities](responsibilities.md) clarifies the boundary between developers, cluster administrators, and stack owners. +- [Troubleshooting And Recovery](troubleshooting-recovery.md) covers common build, deployment, and release failures. +- [Security Guidance](security.md) describes credentials, image supply-chain, and namespace expectations. +- [Naming Conventions](naming-conventions.md) lists derived repository, image, chart, release, and pipeline names. +- [Architecture](architecture.md) shows how the developer path relates to OCI DevOps and OKE. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/glossary.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/glossary.md new file mode 100644 index 000000000..cd758950e --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/glossary.md @@ -0,0 +1,23 @@ +# Glossary + +| Term | Meaning | +| --- | --- | +| Application | A logical product containing one or more components and one shared Kubernetes namespace baseline | +| Component | An independently built, charted, deployed, and released workload inside an application | +| Application baseline | The umbrella Helm chart containing shared namespace resources but no component workloads | +| Application bootstrap | Idempotent pipeline with parallel stages that initialize one application's noprod and prod namespaces and pull secrets | +| Component chart | A standalone Helm chart deployed independently for dev, staging, and prod | +| SHA image | The immutable main-build image tagged with exactly seven Git commit characters | +| Release candidate | A promoted SHA image and Git tag such as `1.0.0-rc.1` | +| Final release | The production image and Git tag derived by removing `-rc.N` | +| Noprod | The physical OKE target shared by non-production application environments and operations configuration | +| Stage catalog | A cluster-admin deployment pipeline whose tool and baseline stages are invoked selectively | +| Dependency wave | A set of independent selected tools that can deploy in parallel before dependent tools | +| Production Helm verification | Production-only deployment stage that checks the final release with `helm status` before Git tagging | +| Decommission pipeline | Manual cluster operation that deletes one tool's supplemental resources and uninstalls its Helm release; production requires approval | +| Supplemental resources | Tool-specific namespaced YAML applied after the tool's Helm chart | +| Cluster baseline | Cluster-scoped YAML applied after selected tool stages | +| Values artifact | An OCI artifact containing Helm values; cluster-admin versions it with the full configuration commit SHA | +| Configuration commit | The exact `cluster-admin` Git commit used to select and deploy cluster changes | +| Template ownership | The model in which Terraform creates starter resources while preserving later user customization | +| Development mode | Internal stack-packaging mode that allows the development stack to refresh template resources | diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/naming-conventions.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/naming-conventions.md new file mode 100644 index 000000000..1379696e2 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/naming-conventions.md @@ -0,0 +1,181 @@ +# Naming Conventions + +The stack keeps the Resource Manager UI small by deriving names from the DevOps project, application, and component. + +```mermaid +flowchart TB + Inputs["Inputs"] --> Project["devops_project_name\noke-devops-starter"] + Inputs --> App["application\nsample-app"] + Inputs --> Component["component\nsample-api"] + + App --> AppRepo["Repository\nsample-app-chart"] + Component --> SourceRepo["Repository\nsample-api"] + + Project --> ImagePrefix["Image prefix\noke-devops-starter/sample-app/sample-api"] + Project --> AppChartPrefix["Umbrella chart prefix\noke-devops-starter/charts/sample-app"] + Project --> ComponentChartPrefix["Component chart prefix\noke-devops-starter/charts/sample-app/sample-api"] + + App --> Namespace["Namespace\nsample-app"] + Component --> DevRelease["Dev release\nsample-api-dev"] + Component --> StagingRelease["Staging release\nsample-api-staging"] + Component --> ProdRelease["Prod release\nsample-api"] +``` + +## Applications And Components + +- Application names must be unique. +- Component names must be globally unique across all applications. +- Names may use lowercase letters, numbers, dots, underscores, and hyphens. + +Default sample: + +- Application: `sample-app` +- Components: `sample-api`, `sample-worker` + +## Repositories + +- Shared pipeline repository: `pipelines` +- Application chart repository: `-chart` +- Component source repository: `` + +Examples: + +- `shop-chart` +- `invoice` +- `orders-chart` +- `checkout` + +## Chart Paths + +- Application umbrella chart path: `` +- Component chart path: `/charts/` + +Examples: + +- `shop` +- `shop/charts/invoice` +- `orders/charts/checkout` + +## OCIR Prefixes + +OCIR content is grouped by DevOps project and application. + +Images: + +```text +// +``` + +Umbrella charts: + +```text +/charts/ +``` + +Component charts: + +```text +/charts// +``` + +Full examples: + +```text +fra.ocir.io//oke-devops-starter/shop/invoice:1.0.0 +oci://fra.ocir.io//oke-devops-starter/charts/shop +oci://fra.ocir.io//oke-devops-starter/charts/shop/invoice +``` + +Generated build specs expose these prefixes as environment variables, such as `application_chart_repo_prefix`, `component_chart_repo_prefix`, and `component_image_repo_prefix`, so teams can customize the convention later inside the pipeline template. + +## Pipelines + +Application pipelines: + +- `-package` +- `-bootstrap` +- `-deploy` + +Component pipelines: + +- `-pr` +- `-build` +- `-dev-deploy` +- `-release-build` +- `-release` + +Cluster administration: + +- Repository: `cluster-admin` +- Configuration build: `cluster-admin-build` +- Pull request validation: `cluster-admin-pr` +- Shared chart mirror: `cluster-admin-mirror-charts` +- Cluster deployment pipeline: `cluster-admin-` +- Cluster tool removal pipeline: `cluster-admin--decommission` +- Cluster orchestrator stage: `Deploy Cluster Changes` +- Tool namespace: `` by default +- Mirrored tool chart: `/charts/cluster-tools/` +- Values artifact path: `cluster-admin//tools//values.yaml` +- Values artifact version: full cluster-admin Git commit SHA +- Deployment plan artifact path: `cluster-admin/deployment-plan.json` +- Deployment plan artifact version: full cluster-admin Git commit SHA +- Tool dependency metadata: `depends_on: []` +- Shared deployment command spec: `cluster-admin-deploy-command-spec` + +## Helm Releases + +Application baseline: + +- Noprod: `-noprod` +- Prod: `` + +Component releases: + +- Dev: `-dev` +- Staging: `-staging` +- Prod: `` + +This avoids names like `-dev-dev` and keeps prod clean. + +## Kubernetes Namespaces + +Default namespace: + +```text + +``` + +`dev` and `staging` component releases share the application namespace in the pre-prod cluster. Prod uses `prod_namespace` if supplied, otherwise the application namespace. + +## Value Artifacts + +Application baseline values: + +- `-values-noprod` +- `-values-prod` + +Component values: + +- `-values-dev` +- `-values-staging` +- `-values-prod` + +Value artifacts contain environment or cluster-specific values. Image tags are passed through deployment parameters from build pipelines. + +Shared component command specs: + +- `component-promote-release-image-command-spec` +- `component-tag-release-commit-command-spec` + +All component pipelines use the generic deployment parameters `component_chart_version`, `image_repository`, and `image_tag`. The promotion stage derives and exports `release_image_tag`, while the production chart derives that same final tag from the RC `image_tag` because OCI Helm artifact substitution cannot consume the command-stage export. + +## Tags And Labels + +OCI DevOps artifacts and pipelines use freeform tags where useful: + +- Application-owned resources: `application = ` +- Component-owned resources: `application = `, `component = ` +- Shared bootstrap resources use purpose and role tags without claiming ownership by one application. +- Shared component delivery command specs use `purpose = component-delivery` and a role tag instead of claiming ownership by one component. + +Kubernetes component resources include an `env` label for dev and staging. Prod uses `env: prod` but does not add `prod` to the resource name. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/operations.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/operations.md new file mode 100644 index 000000000..a4aa198dc --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/operations.md @@ -0,0 +1,108 @@ +# Stack Operations + +This guide is for the team packaging, applying, and validating the Resource Manager stack. For day-to-day OKE tool and cluster-resource administration, start with the [Cluster Operations Guide](cluster-operations.md). + +Before distributing or upgrading the stack, review [Template Ownership And Upgrades](template-ownership.md), [Responsibilities](responsibilities.md), and [Security Guidance](security.md). + +## Applying The Stack + +Build and upload a fresh stack zip when the Terraform templates change. Keep generated local context out of the zip, especially: + +- `AGENT.md` +- `.agents` +- `.git` +- `.terraform` +- local state and plan files + +After apply, Resource Manager creates the starter OCI DevOps resources. Existing application/component DevOps resources ignore user-editable changes, and repository seeders never overwrite an existing path. + +Repository initialization is intentionally create-only: + +- Existing source repositories are adopted without modification. +- Existing shared and application files are preserved. +- Adding an application or component adds only its missing pipeline/chart paths. +- Removing an application or component from the JSON does not delete its Git files, although Terraform can still destroy its OCI resources. +- Template improvements from a newer stack archive are not applied to existing repositories automatically. + +Review and merge newer template changes through Git when a customized repository should adopt them. + +## Application Information + +After a successful apply, the stack's Application Information tab organizes outputs into: + +- Start Here: project, configured applications/components, namespaces, and derived next steps. +- Repositories: component source and application chart repository URLs. +- Build Pipelines: PR, build, release-build, and application package pipeline OCIDs. +- Deployment Pipelines And Environments: namespace initialization, application/component deployment pipelines, and both OKE environments. +- OCIR: application charts, component charts, and component image repositories. +- Cluster Administration: admin repository, shared admin build pipelines, Generic Artifact repository, and cluster/tool deployment pipelines. + +Map-valued outputs are rendered as structured JSON instead of single-line copyable strings. The primary action opens the shared pipelines repository. + +## Suggested Local Checks + +Before uploading the stack: + +```bash +terraform fmt -check -recursive +terraform validate +bash -n repos/pipelines/script/*.sh script/*.sh +helm lint repos/generated/charts// +helm lint repos/generated/charts///charts/ +``` + +The generated chart paths depend on the configured applications. + +## Functional Test Checklist + +For each application: + +1. Run `-bootstrap` and confirm both namespace stages succeed, or invoke only the target stage being tested. +2. Confirm the configured baseline chart version is published in OCIR. +3. Run `-deploy` when testing full baseline promotion. +4. Confirm releases `-noprod` and, after approval, `` exist. + +For each component: + +1. Open a pull request and verify `-pr` succeeds. +2. Merge to `main`. +3. Verify `-build` produces only the 7-character SHA image tag. +4. Verify `-dev-deploy` updates release `-dev`. +5. Run `-release-build` with `release_tag=1.0.0-rc.1`. +6. Verify staging release `-staging`. +7. Review staging, then approve production. +8. Verify prod release ``, final image tag `1.0.0`, and final Git tag `1.0.0`. + +For chart lifecycle: + +1. Modify a component chart and verify only that component chart packages and dev redeploys. +2. Modify an umbrella chart file outside `charts/**` and verify only `-package` and `-deploy` run. + +For cluster administration: + +1. Open a PR changing one cluster baseline or one tool path and verify `cluster-admin-pr` succeeds. +2. Merge the PR and verify `cluster-admin-build` selects only changed stages. +3. Verify `cluster-admin-mirror-charts` skips chart versions already present in OCIR. +4. Verify the values artifact version equals the full Git commit SHA. +5. Verify `cluster-admin-noprod` runs its orchestrator immediately and deploys only selected changes. +6. Verify `cluster-admin-prod` pauses for approval and then runs the same orchestration behavior. +7. Change independent tools together and verify their stages run in parallel; declare `depends_on` and verify the dependent tool waits. +8. Confirm supplemental resources are created only in the configured tool namespace. + +## Cleanup + +To remove application workloads from a test cluster, delete the application namespaces: + +```bash +kubectl delete namespace +``` + +For the default sample: + +```bash +kubectl delete namespace sample-app +``` + +## Capacity Notes + +Dev, staging, and prod releases can all run in the same test cluster if prod points to pre-prod. That is convenient for demos, but it consumes more CPU and memory. If pods remain pending with an `Insufficient cpu` event, scale down old dev or staging deployments, add capacity, or use a separate production cluster. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/quickstart.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/quickstart.md new file mode 100644 index 000000000..70497c4f6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/quickstart.md @@ -0,0 +1,219 @@ +# Quickstart + +This stack is a starting point for customers adopting OCI DevOps with OKE. It creates the repositories, pipelines, artifacts, environments, naming conventions, and IAM wiring that otherwise require substantial OCI-specific configuration. + +Application tests, quality gates, deployment checks, chart values, and operational policies are deliberately templates. Configure the initial shape through Resource Manager, apply the stack, and then let the owning teams evolve the generated repositories and pipelines. + +## What You Need + +Before opening the Resource Manager form, prepare: + +- An OCI compartment for the DevOps project. +- At least one OKE cluster. A separate production cluster is recommended but not required for an evaluation. +- A worker subnet for each cluster that OCI DevOps shell runners can use. +- An optional worker NSG for each cluster. +- An OCI auth token used once to seed the hosted Git repositories. +- A Vault secret containing the OCIR pull credential used by application bootstrap. +- IAM permissions for OCI DevOps, OKE, OCIR, Vault, Logging, and Artifact Registry. + +The OKE API can be public or private. Deployment shell runners always use the private endpoint from the selected worker network. + +## 1. Configure The Stack + +Select **Deploy to Oracle Cloud** in the main README. Resource Manager opens with the Terraform configuration already selected. Work through the form from top to bottom. + +### OCI DevOps + +Select the DevOps compartment and set the project name. The project name becomes part of every generated image and chart path. + +Provide the Git seeding credentials, logging configuration, and notification topic. Enable IAM creation only when the stack should create its dynamic group and policy. + +### Applications + +Define applications and their components as formatted JSON. The default shape is: + +```json +[ + { + "name": "sample-app", + "components": [ + { "name": "sample-api" }, + { "name": "sample-worker" } + ] + } +] +``` + +Use application names for namespaces and umbrella charts. Component names must be globally unique because they become top-level repository, pipeline, chart, and image identities. + +Optional component fields customize chart versions and build specification paths. Leave them out for the standard conventions. + +### OKE Environments + +For both pre-production and production, select: + +- OKE compartment and cluster. +- Network compartment. +- Worker subnet. +- Optional worker NSG. + +If only the pre-production cluster exists, select that same cluster in both the **OKE Pre-Prod Environment** and **OKE Prod Environment** sections. Use its corresponding network compartment, worker subnet, and optional NSG in both sections. This enables the complete pipeline structure without requiring a second cluster. + +When both environments point to one physical cluster, application releases remain distinguishable through their release names, but cluster tools use the same namespaces and Helm release names. Treat this as an evaluation configuration: a prod tool deployment upgrades the same live tool release rather than creating an isolated copy. Use separate clusters for a real production boundary. + +### Cluster Administration + +Leave `enable_cluster_admin` disabled when the stack is only for application developers. + +To create the operations workflow, enable it and define the shared tool catalog: + +```json +{ + "tools": [ + { + "name": "keda", + "repository": "https://kedacore.github.io/charts", + "chart": "keda", + "version": "2.20.1", + "namespace": "keda", + "depends_on": [] + } + ] +} +``` + +The catalog pins chart coordinates and dependency topology. Values remain separate for `noprod` and `prod` in the generated `cluster-admin` repository. + +## 2. Apply And Check Outputs + +Run an Apply job and wait for `SUCCEEDED`. In Application Information, verify that Resource Manager reports: + +- The OCI DevOps project. +- The shared `pipelines` repository. +- One chart repository per application. +- One source repository per component. +- Build, PR, release, bootstrap, and deployment pipelines. +- Noprod and prod OKE environments. +- Cluster-admin resources when the feature is enabled. + +OCI DevOps runner provisioning can take several minutes. A pipeline remaining in `ACCEPTED` or `IN_PROGRESS` while a build or shell runner starts is not, by itself, a failure. + +## 3. Initialize An Application + +Run `-bootstrap` before deploying application charts or components. + +Provide: + +- `registry_username`: the OCIR pull username. +- `pull_password_secret_ocid`: the Vault secret containing the pull credential. +- `secret_name`: normally `ocirsecret`. + +The pipeline has independent noprod and prod stages. Run the whole pipeline to initialize both clusters, or run one stage when only one target is ready. + +Verify the result: + +```bash +kubectl get namespace +kubectl -n get secret ocirsecret +``` + +The secret type must be `kubernetes.io/dockerconfigjson`. + +## 4. Deploy The Application Baseline + +The umbrella chart owns shared namespace resources, not component workloads. + +1. Run `-package` to publish the initial umbrella chart. +2. Let it start `-deploy`. +3. Verify release `-noprod`. +4. Review and approve production. +5. Verify release `` in the prod cluster. + +All component subcharts are disabled in the baseline release. + +## 5. Start Component Development + +Open the generated component repository and replace the sample implementation. + +Before the first real merge: + +1. Implement the component `Dockerfile`; prefer a multi-stage build that produces both `linux/amd64` and `linux/arm64` images. +2. Replace the placeholder `.oci-devops/pull-request-pipeline.yaml` with language-appropriate unit, integration, security, and quality checks. +3. Customize the component Helm chart when the workload needs additional values or Kubernetes resources. +4. Customize the shared build specification only when the standard component build does not fit. + +Then follow the normal path: + +```mermaid +flowchart LR + Branch["Feature branch"] --> PR["Pull request"] + PR --> Tests["-pr"] + Tests --> Merge["Merge to main"] + Merge --> Build["-build"] + Build --> Image["Image tagged with SHA7"] + Image --> Dev["Deploy -dev"] +``` + +The main build creates only the immutable 7-character Git SHA image tag. It does not create a release version tag. + +## 6. Promote A Component + +Run `-release-build` with a strict release candidate such as: + +```text +1.0.0-rc.1 +``` + +The flow: + +1. Resolves `main`, or the optional full `commit_id`. +2. Verifies the corresponding SHA7 image exists. +3. Creates the RC Git tag and retags the image without copying layers locally. +4. Deploys release `-staging` in noprod. +5. Waits for production approval. +6. Retags the RC image as `1.0.0`. +7. Deploys release `` in prod. +8. Creates final Git tag `1.0.0`. +9. Reports the production Helm status. + +Review the staging result before approving production. Release tags are immutable; use a new RC or version rather than moving an existing tag. + +## 7. Start Cluster Operations + +When cluster administration is enabled, clone `cluster-admin` and configure cluster-specific files: + +```text +clusters/ + noprod/ + baseline/ + tools//values.yaml + prod/ + baseline/ + tools//values.yaml +``` + +Use a pull request for every change. After merge: + +- `cluster-admin-build` detects only affected cluster paths. +- Missing pinned charts are mirrored into OCIR. +- Values and deployment plans are published with the full Git commit as their immutable version. +- Noprod changes start immediately. +- Prod changes wait at `cluster-admin-prod` for approval before any mutation. +- Independent tools run in parallel; `depends_on` creates ordered waves. +- Cluster-wide baseline objects run after tools so they may use chart-installed CRDs. + +Changing only `clusters/prod/...` does not redeploy noprod. Changing `catalog/tools.yaml` can select configured tools on both clusters because chart coordinates are shared. + +## Success Checklist + +A first installation is healthy when: + +- Application bootstrap created namespaces and pull secrets on the selected clusters. +- The application baseline has separate noprod and prod Helm releases. +- A component PR pipeline succeeds. +- A main merge produces a SHA7 multi-architecture image and deploys dev. +- A release candidate reaches staging, approval, final image tagging, prod, and final Git tagging. +- Optional cluster tools are installed in their own namespaces with the expected cluster-specific values. +- Later Resource Manager applies do not overwrite repository or pipeline customizations in release mode. + +For failures, continue with [Troubleshooting And Recovery](troubleshooting-recovery.md). For all stack fields, see [Resource Manager Inputs](resource-manager-inputs.md). diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/resource-manager-inputs.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/resource-manager-inputs.md new file mode 100644 index 000000000..0f1ca062c --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/resource-manager-inputs.md @@ -0,0 +1,166 @@ +# Resource Manager Inputs + +The Resource Manager schema is organized around the decisions a developer platform team must make up front. Most names and paths are derived. + +Application configuration supports the [developer workflow](developers.md). The optional cluster-administration configuration supports the independent [cluster operations workflow](cluster-operations.md). OKE, OCI DevOps, and IAM inputs are shared platform concerns. + +```mermaid +flowchart TB + RM["Resource Manager inputs"] + + RM --> DevOps["OCI DevOps\ncompartment, project name, auth token"] + RM --> Apps["Applications\napplication names and component names"] + RM --> Preprod["OKE pre-prod environment\ncluster, network compartment, subnet, optional NSG"] + RM --> Prod["OKE prod environment\ncluster, network compartment, subnet, optional NSG"] + RM --> IAM["IAM\npolicy creation and Vault compartment"] + RM --> ClusterAdmin["Cluster administration\nshared tool topology"] + + DevOps --> DerivedProject["Derived project prefix\n"] + Apps --> DerivedRepos["Derived repositories\n-chart and "] + Apps --> DerivedPipelines["Derived pipelines\n-build, -release, -deploy"] + Preprod --> NoprodEnv["DevOps environment\noke-cluster-noprod"] + Prod --> ProdEnv["DevOps environment\noke-cluster-prod"] +``` + +## OCI DevOps + +Configure: + +- DevOps compartment. +- DevOps project name, default `oke-devops-starter`. +- DevOps project description. +- Auth token used to seed OCI DevOps Code Repositories. +- Notification topic behavior: create a new topic or use an existing topic OCID. +- Logging behavior: create a DevOps service log, then configure log group name, service log name, and retention. +- Namespace-init pull secret name, default `ocirsecret`. + +The project name also becomes part of image and chart prefixes. For example, project `oke-devops-starter`, application `shop`, and component `invoice` produces image prefix `oke-devops-starter/shop/invoice`. + +## Applications + +`applications` is entered as a formatted JSON array in a multiline editor. Keep the normal input small: an application name and one or more component names. Terraform validates and decodes the JSON before passing the strongly typed application list to the DevOps module. + +```json +[ + { + "name": "shop", + "components": [ + { "name": "invoice" } + ] + }, + { + "name": "orders", + "components": [ + { + "name": "checkout", + "build_spec_path": "java/java-build-pipeline.yaml" + }, + { + "name": "fulfillment", + "build_spec_path": "java/java-build-pipeline.yaml" + } + ] + } +] +``` + +Optional fields exist for advanced use: + +- `chart_repository_name` +- `chart_path` +- `chart_version` +- `namespace` +- `prod_namespace` +- `kubernetes_group` +- component `chart_version` +- component `build_spec_path` + +`build_spec_path` is relative to the `pipelines` repository. If omitted, Resource Manager generates and owns `-build-pipeline.yaml`. If explicitly configured, Resource Manager creates the file and any parent folders from the default component template only when the path is missing. The first commit is a starter for the DevOps engineer; subsequent applies never refresh or overwrite it. Multiple components can share a specification such as `java/java-build-pipeline.yaml`. A shared starter identifies every referencing component and should be generalized before all of those component builds are enabled. + +Defaults are derived from the application and component names. Application names must be unique. Component names must be globally unique across all applications. + +Names are validated after their derived suffixes are considered: application names are limited to 46 characters, component names to 45, and namespaces must be unique DNS labels within each cluster. Repository names cannot collide with another application/component or the reserved `pipelines` and `cluster-admin` repositories. The `estimated_devops_resources` output helps platform owners compare the generated topology with OCI service limits; larger-than-recommended topologies produce non-blocking Terraform check warnings. + +Adding an application or component later creates its OCI resources and adds only missing entity-specific repository paths. Existing repository files are treated as developer-owned and are not overwritten by later applies. Explicit custom build-spec paths are add-only and excluded from development refresh. + +## Cluster Administration + +Cluster administration is optional and disabled by default. Set `enable_cluster_admin=true` to create the operations repository, immutable values repository, build pipelines, one orchestrator per cluster, triggers, artifacts, and branch protection. The production orchestrator has a mandatory approval predecessor. The topology editor and help text appear only after the feature is enabled. + +`cluster_admin_artifact_repository_name` optionally sets the OCI Generic Artifact repository display name used for immutable tool values and cluster deployment plans. Leave it empty to derive `-cluster-admin-values`, preserving the existing convention. + +`cluster_administration` is a multiline JSON object defining one tool topology for both physical clusters. A missing namespace defaults to the tool name. Noprod starts immediately, while prod always requires one approval before any deployment stage starts. + +```json +{ + "tools": [ + { + "name": "keda", + "repository": "https://kedacore.github.io/charts", + "chart": "keda", + "version": "2.20.1", + "namespace": "keda", + "depends_on": [] + }, + { + "name": "kube-prometheus", + "repository": "https://prometheus-community.github.io/helm-charts", + "chart": "kube-prometheus-stack", + "version": "87.10.1", + "namespace": "monitoring", + "depends_on": [] + } + ] +} +``` + +The shared list configures both cluster orchestrators and seeds `cluster-admin/catalog/tools.yaml` from the configured repositories, chart names, and versions. Values, supplemental resources, and baseline manifests remain cluster-specific under `clusters/`. + +`repository` accepts either a traditional HTTPS Helm repository or an OCI repository base path. For OCI sources, the mirror constructs `/`: + +```json +{ + "name": "external-dns", + "repository": "oci://registry-1.docker.io/bitnamicharts", + "chart": "external-dns", + "version": "9.0.3", + "namespace": "external-dns", + "depends_on": [] +} +``` + +Public OCI repositories work anonymously. A private source works only when Helm already has credentials for that registry; the stack automatically logs in only to its target OCIR registry. + +For compatibility, existing stack inputs using `noprod.tools` and `prod.tools` are accepted; `noprod.tools` becomes the shared canonical topology. + +Disabling the feature removes Terraform-managed cluster-administration resources. It does not uninstall Helm releases or prune Kubernetes objects previously deployed to either cluster. + +## OKE Pre-Prod Environment + +Configure: + +- OKE pre-prod compartment. +- OKE cluster. +- Network compartment. +- Worker subnet. +- Optional worker NSG. + +The stack uses private OKE endpoints for DevOps shell stages. The worker subnet is therefore mandatory. + +## OKE Prod Environment + +Configure the same shape as pre-prod, but with prod-specific values: + +- OKE prod compartment. +- OKE prod cluster. +- Prod network compartment. +- Prod worker subnet. +- Optional prod worker NSG. + +For early testing, prod can point to the same OKE cluster, subnet, and NSG as pre-prod. + +## IAM + +The IAM section controls whether the stack creates the DevOps dynamic group and policies. The Vault compartment input is used by application bootstrap because the OCIR pull password is read from an OCI Vault secret at deployment time. + +When IAM creation is enabled, you can also configure the generated dynamic group name and policy name. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/responsibilities.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/responsibilities.md new file mode 100644 index 000000000..76d830f60 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/responsibilities.md @@ -0,0 +1,60 @@ +# Responsibilities + +The solution serves three collaborating roles. A person may hold more than one role, but the ownership boundaries remain useful when reviewing changes and diagnosing failures. + +| Capability | Application developer | Cluster administrator | Stack/platform owner | +| --- | --- | --- | --- | +| Component source and `Dockerfile` | Owns | Not involved | Defines starter conventions | +| Component PR tests | Owns | Not involved | Seeds placeholder specification | +| Component Helm chart | Owns | Not involved | Defines starter chart conventions | +| Application baseline chart | Contributes requirements | Reviews cluster impact | Owns initial structure and pipeline | +| Component release approval | Participates | May approve by policy | Configures approval model | +| Application namespace and pull secret | Consumes | May operate | Configures and runs initial setup | +| Cluster tool catalog and values | Not involved | Owns | Bootstraps workflow | +| Tool dependencies and namespaces | Not involved | Owns | Validates configuration model | +| Cluster-wide Kubernetes resources | Not involved | Owns | Bootstraps workflow | +| OCI DevOps shared templates | Contributes requirements | Contributes requirements | Owns and versions | +| Resource Manager stack and inputs | Provides application topology | Provides cluster topology | Owns packaging and apply | +| IAM, Vault access, and network selection | Not involved | Provides requirements | Owns or coordinates | +| Functional-test cleanup | Cleans test application releases | Cleans test tools and resources | Verifies stack resources | + +## Application Developers + +Developers own the behavior and deployability of their components. This includes source code, language-specific tests, the multi-stage `Dockerfile`, and component chart changes. Their normal path begins with a pull request and ends with a promoted component release. + +Developers should not put cluster tools or cluster-scoped objects into component repositories. When a component needs a shared cluster capability, request it from the cluster administrators. + +## Cluster Administrators + +Cluster administrators own the `cluster-admin` repository, pinned tool charts, per-cluster values, supplemental resources, dependencies, and baseline cluster-scoped objects. Their path is organized by physical cluster rather than application environment. + +Cluster administrators do not modify component image tags or run application release promotion. Tool changes and application releases remain independent even when they target the same OKE cluster. + +## Stack And Platform Owners + +Stack owners configure and apply Resource Manager, maintain shared Terraform and pipeline templates, coordinate IAM and networking, and decide when template changes should be adopted by existing repositories. They also own one-time application setup through `-bootstrap` unless that responsibility is explicitly delegated. + +Because release-mode resources become user-owned templates, the stack owner must not assume that reapplying a newer ZIP upgrades customized OCI DevOps resources or repository content. + +## Shared Change Rule + +Use the smallest repository and workflow that owns the change: + +```mermaid +flowchart TB + Change["What is changing?"] --> Runtime{"Component runtime or tests?"} + Runtime -->|Yes| Source[" repository"] + Runtime -->|No| Workload{"Component Kubernetes workload?"} + Workload -->|Yes| ComponentChart["Component chart directory"] + Workload -->|No| AppShared{"Shared application namespace object?"} + AppShared -->|Yes| AppChart["Application baseline chart"] + AppShared -->|No| Cluster{"Cluster tool or cluster-scoped object?"} + Cluster -->|Yes| ClusterAdmin["cluster-admin repository"] + Cluster -->|No| Platform["Stack and shared pipelines"] +``` + +- Component behavior or packaging: component source repository. +- Component Kubernetes workload: component chart directory. +- Shared application namespace object: application baseline chart. +- Kubernetes tool or cluster object: `cluster-admin` repository. +- Delivery convention affecting every team: shared `pipelines` repository and stack template. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/security.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/security.md new file mode 100644 index 000000000..1d0799630 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/security.md @@ -0,0 +1,39 @@ +# Security Guidance + +The stack creates delivery capabilities with access to source repositories, OCIR, Vault, and OKE. Treat pipeline identity and repository review as production security boundaries. + +## Credentials And Secrets + +- Store the OCIR pull password or auth token in OCI Vault and pass only its secret OCID to `-bootstrap`. +- Never commit secret material to application charts, component repositories, cluster values, or supplemental resource YAML. +- Do not use plain Kubernetes `Secret` manifests in the cluster-admin repository; use External Secrets or an equivalent external-secret reference. +- Rotate repository seeding credentials and OCIR tokens according to organizational policy. +- Review the published Terraform configuration before distribution and exclude local state, plans, credentials, logs, private ad hoc test/deployment scripts, `.git`, `.terraform`, `.agents`, and `AGENT.md`. Bundled stack templates and their intentional runtime scripts remain part of the product. + +## IAM + +- Prefer a dedicated dynamic group and narrowly scoped policies for OCI DevOps execution resources. +- Restrict Vault secret access to the configured Vault compartment and only the identities that run namespace initialization. +- Separate stack maintainers, application release approvers, and cluster production approvers where organizational controls require it. +- Review manually customized pipelines before expanding their IAM permissions. + +## Network Access + +- DevOps shell stages use OKE private endpoints and run in the selected worker subnet with the optional worker NSG. +- Keep worker-subnet egress limited to required OCI services, chart sources, and application dependencies. +- Configure pre-prod and prod subnet and NSG inputs independently, even when a temporary test setup points both environments to the same cluster. + +## Supply Chain + +- Pin every cluster tool repository, chart, and version in the Resource Manager tool definition; the generated `catalog/tools.yaml` must not contain floating versions. +- Keep image, chart, and values versions immutable. +- Use multi-stage Dockerfiles to keep compilers, package managers, credentials, and test tools out of runtime images. +- Replace the placeholder PR pipeline with real tests and add language-appropriate dependency and image scanning where required. +- Review the successful staging deployment before production approval. + +## Kubernetes Boundaries + +- Give each application and cluster tool its own namespace. +- Component ServiceAccounts reference the configured OCIR pull secret instead of modifying the namespace default ServiceAccount. +- Supplemental tool resources are restricted to the tool namespace. +- Cluster-scoped baseline changes require particular review because they can affect every namespace and may depend on CRDs installed by tool charts. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/template-ownership.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/template-ownership.md new file mode 100644 index 000000000..a0c9e02ea --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/template-ownership.md @@ -0,0 +1,60 @@ +# Template Ownership And Upgrades + +The stack bootstraps OCI DevOps resources and repositories, but it is not intended to remain authoritative for every user customization. + +## Release Mode + +`./update.sh` creates a release archive by default. + +- Generated DevOps resources retain `ignore_changes = all` where the provider permits it. +- Teams can customize pipelines, stages, artifacts, tags, and repository settings without a later stack apply reverting them. +- Repository seed operations preserve existing paths and add only missing entity-specific content. +- A newer stack ZIP does not automatically upgrade customized pipeline internals or existing repository files. + +Use release mode for distributed stacks that become starting points for developer and administrator ownership. + +## Development Mode + +Build a development archive with: + +```bash +STACK_DEVELOPMENT_MODE=true ./update.sh +``` + +Development mode removes release-only lifecycle ignores from the staged archive and sets the hidden `development_mode` input. It lets the development Resource Manager stack refresh shared template resources while retaining provider-safety ignores and repository ownership protections. + +Use development mode only for stack development and functional testing. It is not a general mechanism for forcing repository content over user changes. + +## Repository Seeding + +| Seed target | Later apply behavior | +| --- | --- | +| Shared `pipelines` files | Seed when empty; preserve existing files | +| Component source repository | Seed when empty; preserve developer content | +| Application baseline chart | Seed when empty | +| New component chart directory | Add only when the path is missing | +| New entity-specific pipeline specification | Add only when the path is missing | +| Explicit component `build_spec_path` | Add the default starter only when missing, creating parent folders automatically; never refresh the file afterward | +| `cluster-admin` repository | Add initial content without replacing existing paths | + +Removing an application, component, or tool from Resource Manager may destroy Terraform-managed OCI resources, but it does not delete the corresponding Git files or live Kubernetes workloads automatically. + +## Adopting A Template Upgrade + +1. Read the newer stack release notes and identify affected Terraform resources, pipeline files, and scripts. +2. Back up or branch customized repositories. +3. Apply infrastructure-compatible changes through Resource Manager. +4. Compare newer repository templates with the user-owned files. +5. Port desired changes through normal pull requests instead of reseeding repositories. +6. Test one application/component or cluster target before broad adoption. +7. Verify custom OCI DevOps stage parameters and tags after the apply. + +## Ownership Test + +Before distributing a release archive: + +1. Apply it once and customize a generated pipeline field. +2. Apply the same archive again. +3. Confirm the customization remains. +4. Add a new application or component and apply again. +5. Confirm only missing resources and repository paths are added. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/troubleshooting-recovery.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/troubleshooting-recovery.md new file mode 100644 index 000000000..ce1400d35 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/docs/troubleshooting-recovery.md @@ -0,0 +1,63 @@ +# Troubleshooting And Recovery + +Start with the first failed pipeline or deployment stage. Later resources often fail only because an expected image, chart, values artifact, namespace, or CRD was not produced earlier. + +```mermaid +flowchart TD + Failure["Pipeline or deployment failed"] --> First["Find the first failed stage"] + First --> Mutation{"Was the target mutated?"} + Mutation -->|No| Fix["Correct input or source and rerun"] + Mutation -->|Yes| Healthy{"Is the previous release healthy?"} + Healthy -->|Yes| Rollback["Roll back to retained deployment snapshot"] + Healthy -->|No| Stabilize["Stop approvals and stabilize manually"] + Rollback --> Verify["Verify cluster and workload health"] + Stabilize --> Verify + Fix --> Verify + Verify --> Record["Record outcome and fix forward through Git"] +``` + +## Application Delivery + +| Symptom | Likely cause | Recovery | +| --- | --- | --- | +| PR pipeline does not start | PR does not target `main`, trigger was customized, or build spec is missing | Inspect the repository trigger and `.oci-devops/pull-request-pipeline.yaml`, then update the PR branch | +| Main build cannot find metadata | `.oci-devops/application.env` is missing or invalid | Restore `component_name` and rerun the build | +| Multi-architecture build fails | Dockerfile cannot build for one target architecture | Test each build target and remove architecture-specific build assumptions | +| Release build reports missing SHA image | Selected commit never completed a main build | Build the commit from `main` or select a commit with an existing `` image | +| RC or final tag already exists | Version was already used | Verify existing tag ownership; use a new RC number or version rather than moving it | +| Pod shows `ImagePullBackOff` | Namespace secret is missing, invalid, or not referenced by the ServiceAccount | Rerun the relevant stage of `-bootstrap` with correct Vault credentials and inspect the component ServiceAccount | +| Production is waiting | Manual approval has not been accepted | Review the successful staging deployment and approve or reject production | +| Production fails after image promotion | Final image exists but Helm deployment failed | Fix the deployment issue and retry safely; do not repoint the immutable version tag | +| Production status stage fails | Helm could not report the completed release | Inspect the production release and workload; deployment and final tagging have already completed, so roll back or fix forward as appropriate | + +If staging succeeds but production is rejected, no production mutation occurs. The RC image and Git tag remain valid for investigation or a later decision. + +## Cluster Administration + +| Symptom | Likely cause | Recovery | +| --- | --- | --- | +| PR validation rejects topology | Unknown dependency, cycle, duplicate, or mismatched namespace | Correct Resource Manager topology and repository `tool.yaml` definitions | +| Chart mirror fails | Catalog source/version is invalid or upstream is unavailable | Verify the pinned public repository and chart version, then rerun | +| Values artifact is missing | Configuration build failed before publication | Fix the validation/publication failure and rerun from the same or a new commit | +| Only part of a dependency wave succeeds | One independently invoked stage failed | Fix the failed target and rerun the configuration build; successful idempotent stages may run again | +| Supplemental resource is rejected | Plain Secret or cross-namespace target | Use an external secret reference and the configured tool namespace | +| Baseline custom resource fails | Required CRD was not installed or tool stage failed | Verify the owning tool chart and dependency order, then rerun the baseline change | +| Prod has no mutation deployment | Approval is pending or rejected | Inspect the approval stage inside `cluster-admin-prod`; its orchestrator starts only after success | + +## Rollback And Drift + +- OCI DevOps deployment snapshots retain the exact chart and values used by that deployment. A rollback does not fetch the latest mutable values definition. +- Image and chart versions should remain immutable so a historical snapshot resolves to the same content. +- Removing YAML from Git does not delete the live Kubernetes object. +- The cluster-admin workflow does not continuously reconcile out-of-band changes. +- Record manual corrections and bring the repository back into agreement through a reviewed pull request. + +## Partial Failure Procedure + +1. Stop additional approvals or manual retries until the failed stage is understood. +2. Record the build run, deployment, Git commit, target cluster, chart version, values version, and image tag. +3. Inspect the target namespace and Helm release without changing unrelated resources. +4. Decide whether to retry, roll back, or fix forward. +5. Reuse a commit only when all immutable artifacts already associated with it are correct. +6. Use a new RC or final version whenever released content must change. +7. Verify workload health and clean test-only resources after recovery. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/locals.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/locals.tf new file mode 100644 index 000000000..ab8628aa4 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/locals.tf @@ -0,0 +1,20 @@ +locals { + git_username = "${data.oci_identity_tenancy.current_tenancy.name}/${data.oci_identity_user.current_user.name}" + application_config = jsondecode(var.applications) + application_count = length(local.application_config) + component_count = length(flatten([ + for application in local.application_config : application.components + ])) + estimated_devops_resources = { + repositories = 1 + local.application_count + local.component_count + (var.enable_cluster_admin ? 1 : 0) + build_pipelines = local.application_count + (3 * local.component_count) + ( + var.enable_cluster_admin ? 3 : 0 + ) + deployment_pipelines = (2 * local.application_count) + (2 * local.component_count) + ( + var.enable_cluster_admin ? 4 : 0 + ) + deployment_stages = 1 + (5 * local.application_count) + (9 * local.component_count) + ( + var.enable_cluster_admin ? 6 : 0 + ) + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/main.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/main.tf new file mode 100644 index 000000000..78f56d71d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/main.tf @@ -0,0 +1,51 @@ +module "devops" { + source = "./modules/devops" + + compartment_id = coalesce(var.devops_compartment_id, var.compartment_ocid) + region = var.region + tenancy_id = var.tenancy_ocid + git_username = local.git_username + git_password = var.auth_token + create_notification_topic = var.create_notification_topic + notification_topic_id = var.notification_topic_id + notification_topic_name = var.notification_topic_name + notification_topic_description = var.notification_topic_description + devops_project_name = var.devops_project_name + devops_project_description = var.devops_project_description + devops_log_group_name = var.devops_log_group_name + devops_log_group_description = var.devops_log_group_description + devops_log_name = var.devops_log_name + devops_log_is_enabled = var.devops_log_is_enabled + devops_log_retention_period_in_days = var.devops_log_retention_period_in_days + oke_cluster_id = var.oke_cluster_id + oke_worker_subnet_id = var.oke_worker_subnet_id + oke_worker_nsg_id = var.oke_worker_nsg_id + prod_oke_cluster_id = var.prod_oke_cluster_id + prod_oke_compartment_id = var.prod_oke_compartment_id + prod_oke_worker_subnet_id = var.prod_oke_worker_subnet_id + prod_oke_worker_nsg_id = var.prod_oke_worker_nsg_id + namespace_init_secret_name = var.namespace_init_secret_name + applications = local.application_config + enable_cluster_admin = var.enable_cluster_admin + cluster_admin_artifact_repository_name = var.cluster_admin_artifact_repository_name + cluster_administration = jsondecode(var.cluster_administration) + development_mode = var.development_mode +} + +module "iam" { + source = "./modules/iam" + count = var.create_iam ? 1 : 0 + + compartment_id = coalesce(var.devops_compartment_id, var.compartment_ocid) + iam_domain_id = var.devops_iam_domain_id + network_compartment_id = coalesce(var.network_compartment_id, var.compartment_ocid) + prod_network_compartment_id = var.prod_network_compartment_id + oke_compartment_id = coalesce(var.oke_compartment_id, var.compartment_ocid) + prod_oke_compartment_id = var.prod_oke_compartment_id + secret_compartment_id = coalesce(var.namespace_init_secret_compartment_id, var.compartment_ocid) + devops_policy_name = var.devops_policy_name + dynamic_group_name = var.devops_dynamic_group_name + providers = { + oci = oci.home + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/artifacts.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/artifacts.tf new file mode 100644 index 000000000..365e887e6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/artifacts.tf @@ -0,0 +1,252 @@ +resource "oci_devops_deploy_artifact" "application_chart" { + for_each = local.applications_by_name + + argument_substitution_mode = "NONE" + deploy_artifact_type = "HELM_CHART" + description = "Packaged ${each.value.name} application Helm chart in OCIR" + display_name = "${each.value.name}-chart" + freeform_tags = { + application = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + chart_url = each.value.ocir_chart + deploy_artifact_source_type = "HELM_CHART" + deploy_artifact_version = "$${chart_version}" + + helm_verification_key_source { + verification_key_source_type = "NONE" + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "component_chart" { + for_each = local.components_by_name + + argument_substitution_mode = "NONE" + deploy_artifact_type = "HELM_CHART" + description = "Packaged ${each.value.name} Helm chart in OCIR" + display_name = "${each.value.name}-chart" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + chart_url = each.value.ocir_chart + deploy_artifact_source_type = "HELM_CHART" + deploy_artifact_version = "$${component_chart_version}" + + helm_verification_key_source { + verification_key_source_type = "NONE" + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "application_baseline_values" { + for_each = local.applications_by_name + + argument_substitution_mode = "SUBSTITUTE_PLACEHOLDERS" + deploy_artifact_type = "GENERIC_FILE" + description = "${local.baseline_cluster_name} cluster baseline values for the ${each.value.name} Helm release" + display_name = "${each.value.name}-values-${local.baseline_cluster_name}" + freeform_tags = { + application = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/application-baseline-values.yaml.tpl", {})) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "application_prod_values" { + for_each = local.applications_by_name + + argument_substitution_mode = "SUBSTITUTE_PLACEHOLDERS" + deploy_artifact_type = "GENERIC_FILE" + description = "prod cluster baseline values for the ${each.value.name} Helm release" + display_name = "${each.value.name}-values-prod" + freeform_tags = { + application = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/application-baseline-values.yaml.tpl", {})) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "component_values" { + for_each = local.component_environment_pairs + + argument_substitution_mode = "SUBSTITUTE_PLACEHOLDERS" + deploy_artifact_type = "GENERIC_FILE" + description = "${each.value.environment} values for the ${each.value.name} Helm release" + display_name = "${each.value.name}-values-${each.value.environment}" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/component-values.yaml.tpl", { + application_chart_name = each.value.application_name + component_name = each.value.name + environment = each.value.environment + image_tag_parameter = "image_tag" + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "component_prod_values" { + for_each = local.components_by_name + + argument_substitution_mode = "SUBSTITUTE_PLACEHOLDERS" + deploy_artifact_type = "GENERIC_FILE" + description = "prod values for the ${each.value.name} Helm release" + display_name = "${each.value.name}-values-prod" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/component-values.yaml.tpl", { + application_chart_name = each.value.application_name + component_name = each.value.name + environment = "prod" + image_tag_parameter = "image_tag" + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "application_bootstrap_command_spec" { + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Initializes an application namespace and OCIR image pull secret on noprod or prod" + display_name = "application-bootstrap-command-spec" + freeform_tags = { + purpose = "application-bootstrap" + role = "namespace-initialization" + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/application-bootstrap-command-spec.yaml.tpl", { + kube_endpoint = local.kube_endpoint + noprod_oke_cluster_id = var.oke_cluster_id + prod_oke_cluster_id = local.prod_oke_cluster_id + region = var.region + region_key = local.region_key + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "component_verify_deployment_command_spec" { + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Reports a completed production component Helm release status" + display_name = "component-prod-status-command-spec" + freeform_tags = { + purpose = "component-delivery" + role = "deployment-status" + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/verify-component-production-command-spec.yaml.tpl", { + kube_endpoint = local.kube_endpoint + prod_oke_cluster_id = local.prod_oke_cluster_id + region = var.region + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "promote_release_image_command_spec" { + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Promotes an approved component RC image tag to the final release tag" + display_name = "component-promote-release-image-command-spec" + freeform_tags = { + purpose = "component-delivery" + role = "release-image-promotion" + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/promote-release-image-command-spec.yaml.tpl", { + region = var.region + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "tag_release_commit_command_spec" { + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Tags a released component source commit after production deployment" + display_name = "component-tag-release-commit-command-spec" + freeform_tags = { + purpose = "component-delivery" + role = "release-commit-tagging" + } + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/tag-release-commit-command-spec.yaml.tpl", { + region = var.region + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/build_pipelines.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/build_pipelines.tf new file mode 100644 index 000000000..2beb64f72 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/build_pipelines.tf @@ -0,0 +1,342 @@ +resource "oci_devops_build_pipeline" "application_delivery" { + for_each = local.components_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-build" + description = "Builds ${each.value.name} images when source changes and packages the component chart when chart files change" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "application_delivery" { + for_each = local.components_by_name + + build_pipeline_id = oci_devops_build_pipeline.application_delivery[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "${title(replace(each.value.name, "-", " "))} Build" + description = "Conditionally builds the component image and packages the component chart" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + build_spec_file = each.value.build_spec_path + primary_build_source = "pipelines" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.application_delivery[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "pipelines" + repository_id = oci_devops_repository.platform_pipelines.id + repository_url = oci_devops_repository.platform_pipelines.http_url + } + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = each.value.name + repository_id = oci_devops_repository.application_source[each.key].id + repository_url = oci_devops_repository.application_source[each.key].http_url + } + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "application-chart" + repository_id = oci_devops_repository.application_chart[each.value.application_name].id + repository_url = oci_devops_repository.application_chart[each.value.application_name].http_url + } + } + + depends_on = [ + null_resource.seed_platform_entities, + null_resource.seed_application_source, + null_resource.seed_application_chart_components + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "trigger_dev_deployment" { + for_each = local.components_by_name + + build_pipeline_id = oci_devops_build_pipeline.application_delivery[each.key].id + build_pipeline_stage_type = "TRIGGER_DEPLOYMENT_PIPELINE" + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:dev"].id + display_name = "${title(replace(each.value.name, "-", " "))} Dev Deploy" + description = "Triggers the dev component deployment after a successful build" + is_pass_all_parameters_enabled = true + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline_stage.application_delivery[each.key].id + } + } + + depends_on = [ + oci_devops_deploy_stage.deploy_component + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline" "application_baseline_package" { + for_each = local.applications_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-package" + description = "Packages the ${each.value.name} application baseline chart and deploys it to the noprod cluster" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "application_baseline_package" { + for_each = local.applications_by_name + + build_pipeline_id = oci_devops_build_pipeline.application_baseline_package[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "${title(replace(each.value.name, "-", " "))} Package" + description = "Packages the ${each.value.name} umbrella baseline chart" + freeform_tags = { + application = each.value.name + } + build_spec_file = "${each.value.name}-package-pipeline.yaml" + primary_build_source = "pipelines" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.application_baseline_package[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "pipelines" + repository_id = oci_devops_repository.platform_pipelines.id + repository_url = oci_devops_repository.platform_pipelines.http_url + } + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "application-chart" + repository_id = oci_devops_repository.application_chart[each.key].id + repository_url = oci_devops_repository.application_chart[each.key].http_url + } + } + + depends_on = [ + null_resource.seed_platform_entities, + null_resource.seed_application_chart_baseline + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "trigger_application_baseline_deployment" { + for_each = local.applications_by_name + + build_pipeline_id = oci_devops_build_pipeline.application_baseline_package[each.key].id + build_pipeline_stage_type = "TRIGGER_DEPLOYMENT_PIPELINE" + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_application[each.key].id + display_name = "${title(replace(each.value.name, "-", " "))} Noprod Deploy" + description = "Triggers the noprod application baseline deployment after packaging" + is_pass_all_parameters_enabled = true + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline_stage.application_baseline_package[each.key].id + } + } + + depends_on = [ + oci_devops_deploy_stage.deploy_application + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline" "application_pull_request" { + for_each = local.components_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-pr" + description = "Runs component-owned pull request validation for ${each.value.name}" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "application_pull_request" { + for_each = local.components_by_name + + build_pipeline_id = oci_devops_build_pipeline.application_pull_request[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "${title(replace(each.value.name, "-", " "))} Pull Request" + description = "Runs the component-owned pull request build spec" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + build_spec_file = ".oci-devops/pull-request-pipeline.yaml" + primary_build_source = each.value.name + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.application_pull_request[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = each.value.name + repository_id = oci_devops_repository.application_source[each.key].id + repository_url = oci_devops_repository.application_source[each.key].http_url + } + } + + depends_on = [ + null_resource.seed_application_source + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline" "release_application" { + for_each = local.components_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-release-build" + description = "Creates a source Git tag and retags the matching SHA image" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + + build_pipeline_parameters { + items { + name = "release_tag" + default_value = "CHANGE_ME" + description = "Strict SemVer release candidate tag, for example 0.1.0-rc.1" + } + items { + name = "commit_id" + default_value = "CHANGE_ME" + description = "Optional full commit SHA to release. When unset, the current source main commit is used." + } + } + + lifecycle { + # OCI pipeline parameters are intentionally preserved as a complete contract. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "release_application" { + for_each = local.components_by_name + + build_pipeline_id = oci_devops_build_pipeline.release_application[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "${title(replace(each.value.name, "-", " "))} Release" + description = "Creates the source Git tag and retags the SHA image" + build_spec_file = "${each.value.name}-release-pipeline.yaml" + primary_build_source = "pipelines" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.release_application[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "pipelines" + repository_id = oci_devops_repository.platform_pipelines.id + repository_url = oci_devops_repository.platform_pipelines.http_url + } + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = each.value.name + repository_id = oci_devops_repository.application_source[each.key].id + repository_url = oci_devops_repository.application_source[each.key].http_url + } + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "application-chart" + repository_id = oci_devops_repository.application_chart[each.value.application_name].id + repository_url = oci_devops_repository.application_chart[each.value.application_name].http_url + } + } + + depends_on = [ + null_resource.seed_platform_entities, + null_resource.seed_application_source, + null_resource.seed_application_chart_components + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "trigger_staging_deployment" { + for_each = local.components_by_name + + build_pipeline_id = oci_devops_build_pipeline.release_application[each.key].id + build_pipeline_stage_type = "TRIGGER_DEPLOYMENT_PIPELINE" + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + display_name = "${title(replace(each.value.name, "-", " "))} Release Pipeline Trigger" + description = "Triggers the component release deployment pipeline after a successful release build" + is_pass_all_parameters_enabled = true + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline_stage.release_application[each.key].id + } + } + + depends_on = [ + oci_devops_deploy_stage.deploy_component + ] + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_artifacts.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_artifacts.tf new file mode 100644 index 000000000..a93e0688a --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_artifacts.tf @@ -0,0 +1,57 @@ +resource "oci_devops_deploy_artifact" "cluster_admin_deploy_command" { + for_each = local.cluster_admin_singleton + + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Deploys a validated cluster-tool change plan" + display_name = "cluster-admin-deploy-command-spec" + freeform_tags = merge(local.cluster_admin_tags, { + role = "deploy-command" + }) + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/cluster-admin-deploy-command-spec.yaml.tpl", { + artifact_repository_id = oci_artifacts_repository.cluster_admin_values[each.key].id + chart_prefix = "${local.project_repo_prefix}/charts/cluster-tools" + kube_endpoint = local.kube_endpoint + region = var.region + registry = "${local.region_key}.ocir.io" + repository_id = oci_devops_repository.cluster_admin[each.key].id + tenancy_namespace = local.namespace + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_artifact" "cluster_admin_decommission_command" { + for_each = local.cluster_admin_singleton + + argument_substitution_mode = "NONE" + deploy_artifact_type = "COMMAND_SPEC" + description = "Explicitly removes supplemental resources and uninstalls one cluster tool" + display_name = "cluster-admin-decommission-command-spec" + freeform_tags = merge(local.cluster_admin_tags, { + role = "decommission" + }) + project_id = oci_devops_project.devops_project.id + + deploy_artifact_source { + base64encoded_content = base64encode(templatefile("${path.root}/templates/cluster-admin-decommission-command-spec.yaml.tpl", { + kube_endpoint = local.kube_endpoint + noprod_oke_cluster_id = var.oke_cluster_id + prod_oke_cluster_id = local.prod_oke_cluster_id + region = var.region + repository_id = oci_devops_repository.cluster_admin[each.key].id + })) + deploy_artifact_source_type = "INLINE" + } + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_build_pipelines.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_build_pipelines.tf new file mode 100644 index 000000000..7b4a9f2f3 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_build_pipelines.tf @@ -0,0 +1,152 @@ +resource "oci_devops_build_pipeline" "cluster_admin_build" { + for_each = local.cluster_admin_singleton + + project_id = oci_devops_project.devops_project.id + display_name = "cluster-admin-build" + description = "Validates changed cluster configuration, publishes immutable plans and values, and dispatches cluster orchestrators" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "cluster_admin_build" { + for_each = local.cluster_admin_singleton + + build_pipeline_id = oci_devops_build_pipeline.cluster_admin_build[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "Cluster Admin Build" + description = "Validates and dispatches changed cluster administration stages in dependency waves" + build_spec_file = ".oci-devops/build-pipeline.yaml" + primary_build_source = "cluster-admin" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + freeform_tags = merge(local.cluster_admin_tags, { + role = "configuration-dispatch" + }) + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.cluster_admin_build[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "cluster-admin" + repository_id = oci_devops_repository.cluster_admin[each.key].id + repository_url = oci_devops_repository.cluster_admin[each.key].http_url + } + } + + depends_on = [null_resource.seed_cluster_admin_entities] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline" "cluster_admin_mirror" { + for_each = local.cluster_admin_singleton + + project_id = oci_devops_project.devops_project.id + display_name = "cluster-admin-mirror-charts" + description = "Mirrors all missing public Kubernetes tool chart versions from the cluster-admin catalog into OCIR" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "cluster_admin_mirror" { + for_each = local.cluster_admin_singleton + + build_pipeline_id = oci_devops_build_pipeline.cluster_admin_mirror[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "Mirror Cluster Tool Charts" + description = "Mirrors missing pinned public Helm charts into OCIR" + build_spec_file = ".oci-devops/mirror-charts-pipeline.yaml" + primary_build_source = "cluster-admin" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + freeform_tags = merge(local.cluster_admin_tags, { + role = "chart-mirror" + }) + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.cluster_admin_mirror[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "cluster-admin" + repository_id = oci_devops_repository.cluster_admin[each.key].id + repository_url = oci_devops_repository.cluster_admin[each.key].http_url + } + } + + depends_on = [null_resource.seed_cluster_admin_entities] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline" "cluster_admin_pr" { + for_each = local.cluster_admin_singleton + + project_id = oci_devops_project.devops_project.id + display_name = "cluster-admin-pr" + description = "Validates cluster administration pull requests without accessing an OKE cluster" + + lifecycle { + # OCI rejects some updates to parameterless pipeline containers. + ignore_changes = all + } +} + +resource "oci_devops_build_pipeline_stage" "cluster_admin_pr" { + for_each = local.cluster_admin_singleton + + build_pipeline_id = oci_devops_build_pipeline.cluster_admin_pr[each.key].id + build_pipeline_stage_type = "BUILD" + display_name = "Cluster Admin Pull Request" + description = "Validates the cluster tool catalog and per-cluster configuration" + build_spec_file = ".oci-devops/pull-request-pipeline.yaml" + primary_build_source = "cluster-admin" + image = "OL8_X86_64_STANDARD_10" + stage_execution_timeout_in_seconds = 36000 + freeform_tags = merge(local.cluster_admin_tags, { + role = "pull-request-validation" + }) + + build_pipeline_stage_predecessor_collection { + items { + id = oci_devops_build_pipeline.cluster_admin_pr[each.key].id + } + } + + build_source_collection { + items { + connection_type = "DEVOPS_CODE_REPOSITORY" + branch = "main" + name = "cluster-admin" + repository_id = oci_devops_repository.cluster_admin[each.key].id + repository_url = oci_devops_repository.cluster_admin[each.key].http_url + } + } + + depends_on = [null_resource.seed_cluster_admin_entities] + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_deploy_pipelines.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_deploy_pipelines.tf new file mode 100644 index 000000000..b02fcaa64 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_deploy_pipelines.tf @@ -0,0 +1,225 @@ +resource "oci_devops_deploy_pipeline" "cluster_admin" { + for_each = local.cluster_admin_clusters + + project_id = oci_devops_project.devops_project.id + display_name = "cluster-admin-${each.key}" + description = "Applies the cluster baseline and Kubernetes tool DAG to the ${each.key} OKE cluster" + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "cluster-pipeline" + }) + + deploy_pipeline_parameters { + items { + name = "config_commit" + default_value = "" + description = "Exact cluster-admin Git commit containing baseline and supplemental resources" + } + + items { + name = "cluster_id" + default_value = each.value.cluster_id + description = "OKE cluster OCID used by shell deployment stages" + } + + items { + name = "cluster_name" + default_value = each.key + description = "Cluster configuration directory name" + } + + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_pipeline" "cluster_admin_decommission" { + for_each = local.cluster_admin_clusters + + project_id = oci_devops_project.devops_project.id + display_name = "cluster-admin-${each.key}-decommission" + description = "Manually uninstalls one cluster tool from the ${each.key} OKE cluster" + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "decommission-pipeline" + }) + + deploy_pipeline_parameters { + items { + name = "tool_name" + default_value = "" + description = "Tool and Helm release name to remove" + } + items { + name = "tool_namespace" + default_value = "" + description = "Namespace containing the tool release and supplemental resources" + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "cluster_admin_approval" { + for_each = local.cluster_admin_approval_clusters + + approval_policy { + approval_policy_type = "COUNT_BASED_APPROVAL" + number_of_approvals_required = 1 + } + + deploy_pipeline_id = oci_devops_deploy_pipeline.cluster_admin[each.key].id + deploy_stage_type = "MANUAL_APPROVAL" + description = "Approve cluster administration changes for the ${each.key} OKE cluster" + display_name = "Approve ${title(each.key)} Cluster Changes" + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "approval" + }) + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_pipeline.cluster_admin[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "cluster_admin_orchestrator" { + for_each = local.cluster_admin_clusters + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.cluster_admin_deploy_command["enabled"].id + deploy_pipeline_id = oci_devops_deploy_pipeline.cluster_admin[each.key].id + deploy_stage_type = "SHELL" + description = "Deploy the selected cluster administration change plan to ${each.key}" + display_name = "Deploy ${title(each.key)} Cluster Changes" + timeout_in_seconds = 10000 + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "cluster-orchestrator" + }) + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 2 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = each.value.subnet_id + nsg_ids = each.value.nsg_ids + } + } + + deploy_stage_predecessor_collection { + dynamic "items" { + for_each = each.value.approval_required ? [1] : [] + content { + id = oci_devops_deploy_stage.cluster_admin_approval[each.key].id + } + } + + dynamic "items" { + for_each = each.value.approval_required ? [] : [1] + content { + id = oci_devops_deploy_pipeline.cluster_admin[each.key].id + } + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "cluster_admin_decommission_approval" { + for_each = local.cluster_admin_approval_clusters + + approval_policy { + approval_policy_type = "COUNT_BASED_APPROVAL" + number_of_approvals_required = 1 + } + + deploy_pipeline_id = oci_devops_deploy_pipeline.cluster_admin_decommission[each.key].id + deploy_stage_type = "MANUAL_APPROVAL" + description = "Approve explicit removal of a tool from the ${each.key} OKE cluster" + display_name = "Approve ${title(each.key)} Tool Decommission" + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "decommission-approval" + }) + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_pipeline.cluster_admin_decommission[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "cluster_admin_decommission" { + for_each = local.cluster_admin_clusters + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.cluster_admin_decommission_command["enabled"].id + deploy_pipeline_id = oci_devops_deploy_pipeline.cluster_admin_decommission[each.key].id + deploy_stage_type = "SHELL" + description = "Delete supplemental resources and uninstall one Helm release from ${each.key}" + display_name = "Decommission ${title(each.key)} Tool" + timeout_in_seconds = 1200 + freeform_tags = merge(local.cluster_admin_tags, { + cluster = each.key + role = "decommission" + }) + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 2 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = each.value.subnet_id + nsg_ids = each.value.nsg_ids + } + } + + deploy_stage_predecessor_collection { + dynamic "items" { + for_each = each.value.approval_required ? [1] : [] + content { + id = oci_devops_deploy_stage.cluster_admin_decommission_approval[each.key].id + } + } + + dynamic "items" { + for_each = each.value.approval_required ? [] : [1] + content { + id = oci_devops_deploy_pipeline.cluster_admin_decommission[each.key].id + } + } + } + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_repositories.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_repositories.tf new file mode 100644 index 000000000..eb42558bc --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/cluster_admin_repositories.tf @@ -0,0 +1,280 @@ +resource "oci_devops_repository" "cluster_admin" { + for_each = local.cluster_admin_singleton + + name = "cluster-admin" + project_id = oci_devops_project.devops_project.id + description = "Cluster administrator-owned Kubernetes resources, tool catalog, and per-cluster Helm values" + repository_type = "HOSTED" + freeform_tags = merge(local.cluster_admin_tags, { + role = "configuration-repository" + }) + + lifecycle { + ignore_changes = all + } +} + +resource "oci_artifacts_repository" "cluster_admin_values" { + for_each = local.cluster_admin_singleton + + compartment_id = var.compartment_id + display_name = trimspace(var.cluster_admin_artifact_repository_name) != "" ? ( + var.cluster_admin_artifact_repository_name + ) : "${local.project_repo_prefix}-cluster-admin-values" + description = "Immutable per-commit Helm values and cluster plans published by cluster administration" + is_immutable = true + repository_type = "GENERIC" + freeform_tags = merge(local.cluster_admin_tags, { + role = "values-repository" + }) + + lifecycle { + ignore_changes = all + } +} + +resource "local_file" "cluster_admin_readme" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/README.md" + content = templatefile("${path.root}/templates/cluster-admin-README.md.tpl", { + project_name = var.devops_project_name + }) +} + +resource "local_file" "cluster_admin_catalog" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/catalog/tools.yaml" + content = templatefile("${path.root}/templates/cluster-admin-tools.yaml.tpl", { + tools = local.cluster_admin_tools + }) +} + +resource "local_file" "cluster_admin_build_spec" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/.oci-devops/build-pipeline.yaml" + content = templatefile("${path.root}/templates/cluster-admin-build-pipeline.yaml.tpl", { + artifact_repository_id = oci_artifacts_repository.cluster_admin_values[each.key].id + compartment_id = var.compartment_id + mirror_pipeline_id = oci_devops_build_pipeline.cluster_admin_mirror[each.key].id + project_id = oci_devops_project.devops_project.id + region = var.region + region_key = local.region_key + tenancy_namespace = local.namespace + chart_repo_prefix = "${local.project_repo_prefix}/charts/cluster-tools" + }) +} + +resource "local_file" "cluster_admin_mirror_spec" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/.oci-devops/mirror-charts-pipeline.yaml" + content = templatefile("${path.root}/templates/cluster-admin-mirror-pipeline.yaml.tpl", { + compartment_id = var.compartment_id + region = var.region + region_key = local.region_key + tenancy_namespace = local.namespace + chart_repo_prefix = "${local.project_repo_prefix}/charts/cluster-tools" + }) +} + +resource "local_file" "cluster_admin_pr_spec" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/.oci-devops/pull-request-pipeline.yaml" + content = templatefile("${path.root}/templates/cluster-admin-pr-pipeline.yaml.tpl", {}) +} + +resource "local_file" "cluster_admin_baseline_example" { + for_each = local.cluster_admin_clusters + + filename = "${path.root}/${local.cluster_admin_repo_path}/clusters/${each.key}/baseline/README.md" + content = "# ${title(each.key)} cluster baseline\n\nPlace cluster-scoped Kubernetes YAML here. Removing a file does not delete the live object.\n" +} + +resource "local_file" "cluster_admin_tool_values" { + for_each = local.cluster_admin_tool_pairs + + filename = "${path.root}/${local.cluster_admin_repo_path}/clusters/${each.value.cluster_name}/tools/${each.value.tool_name}/values.yaml" + content = "# ${each.value.cluster_name} values for ${each.value.tool_name}\n{}\n" +} + +resource "local_file" "cluster_admin_tool_metadata" { + for_each = local.cluster_admin_tool_pairs + + filename = "${path.root}/${local.cluster_admin_repo_path}/clusters/${each.value.cluster_name}/tools/${each.value.tool_name}/tool.yaml" + content = yamlencode({ + depends_on = each.value.dependencies + name = each.value.tool_name + namespace = each.value.namespace + }) +} + +resource "local_file" "cluster_admin_tool_resources_readme" { + for_each = local.cluster_admin_tool_pairs + + filename = "${path.root}/${local.cluster_admin_repo_path}/clusters/${each.value.cluster_name}/tools/${each.value.tool_name}/resources/README.md" + content = "# Supplemental resources\n\nPlace namespaced Kubernetes YAML for `${each.value.namespace}` here. Plain Kubernetes Secret objects are rejected; use ExternalSecret resources.\n" +} + +resource "local_file" "cluster_admin_validate_script" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/script/validate-config.py" + content = file("${path.root}/repos/cluster-admin/script/validate-config.py") +} + +resource "local_file" "cluster_admin_mirror_script" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/script/mirror-charts.py" + content = file("${path.root}/repos/cluster-admin/script/mirror-charts.py") +} + +resource "local_file" "cluster_admin_publish_script" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/script/publish-and-dispatch.py" + content = file("${path.root}/repos/cluster-admin/script/publish-and-dispatch.py") +} + +resource "local_file" "cluster_admin_deploy_script" { + for_each = local.cluster_admin_singleton + + filename = "${path.root}/${local.cluster_admin_repo_path}/script/deploy-cluster.py" + content = file("${path.root}/repos/cluster-admin/script/deploy-cluster.py") +} + +resource "null_resource" "seed_cluster_admin" { + for_each = local.cluster_admin_singleton + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.cluster_admin[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", ["README.md", "catalog", ".oci-devops", "script"]) + SOURCE_REPO = "/${local.cluster_admin_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.cluster_admin[each.key].id + seed_contract = "cluster-admin-shared-v1" + } + + depends_on = [ + local_file.cluster_admin_readme, + local_file.cluster_admin_catalog, + local_file.cluster_admin_build_spec, + local_file.cluster_admin_mirror_spec, + local_file.cluster_admin_pr_spec, + local_file.cluster_admin_validate_script, + local_file.cluster_admin_mirror_script, + local_file.cluster_admin_publish_script, + local_file.cluster_admin_deploy_script + ] +} + +resource "null_resource" "sync_cluster_admin_catalog" { + for_each = local.cluster_admin_singleton + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.cluster_admin[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "refresh" + SEED_PATHS = "catalog/tools.yaml" + SOURCE_REPO = "/${local.cluster_admin_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.cluster_admin[each.key].id + catalog_hash = sha256(local_file.cluster_admin_catalog[each.key].content) + } + + depends_on = [null_resource.seed_cluster_admin] +} + +resource "null_resource" "seed_cluster_admin_entities" { + for_each = local.cluster_admin_singleton + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.cluster_admin[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", concat( + [for cluster_name in sort(keys(local.cluster_admin_clusters)) : "clusters/${cluster_name}/baseline"], + [ + for key in sort(keys(local.cluster_admin_tool_pairs)) : + "clusters/${local.cluster_admin_tool_pairs[key].cluster_name}/tools/${local.cluster_admin_tool_pairs[key].tool_name}" + ] + )) + SOURCE_REPO = "/${local.cluster_admin_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.cluster_admin[each.key].id + targets = join(",", sort(keys(local.cluster_admin_tool_pairs))) + } + + depends_on = [ + null_resource.sync_cluster_admin_catalog, + local_file.cluster_admin_baseline_example, + local_file.cluster_admin_tool_metadata, + local_file.cluster_admin_tool_values, + local_file.cluster_admin_tool_resources_readme + ] +} + +resource "null_resource" "refresh_cluster_admin_development" { + for_each = var.development_mode ? local.cluster_admin_singleton : {} + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.cluster_admin[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + REMOVE_PATHS = "script/deploy-production.py" + SEED_MODE = "refresh" + SEED_PATHS = join("\n", ["README.md", ".oci-devops", "script"]) + SOURCE_REPO = "/${local.cluster_admin_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.cluster_admin[each.key].id + content_hash = sha256(join("", [ + local_file.cluster_admin_readme[each.key].content, + local_file.cluster_admin_build_spec[each.key].content, + local_file.cluster_admin_mirror_spec[each.key].content, + local_file.cluster_admin_pr_spec[each.key].content, + local_file.cluster_admin_validate_script[each.key].content, + local_file.cluster_admin_mirror_script[each.key].content, + local_file.cluster_admin_publish_script[each.key].content, + local_file.cluster_admin_deploy_script[each.key].content, + ])) + } + + depends_on = [null_resource.sync_cluster_admin_catalog] +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/data.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/data.tf new file mode 100644 index 000000000..8634f10aa --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/data.tf @@ -0,0 +1,15 @@ +data "oci_artifacts_container_configuration" "ocir_config" { + compartment_id = var.compartment_id +} + +data "oci_identity_region_subscriptions" "region_subscriptions" { + tenancy_id = var.tenancy_id +} + +data "oci_containerengine_cluster" "oke_cluster" { + cluster_id = var.oke_cluster_id +} + +data "oci_containerengine_cluster" "prod_oke_cluster" { + cluster_id = local.prod_oke_cluster_id +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/deploy_pipelines.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/deploy_pipelines.tf new file mode 100644 index 000000000..94dfc3dbf --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/deploy_pipelines.tf @@ -0,0 +1,443 @@ +resource "oci_devops_deploy_pipeline" "deploy_application" { + for_each = local.applications_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-deploy" + description = "Promotes the ${each.value.name} namespace baseline through noprod approval and production" + freeform_tags = { + application = each.value.name + } + + deploy_pipeline_parameters { + items { + name = "chart_version" + default_value = each.value.chart_version + description = "Application baseline chart version to deploy" + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_pipeline" "deploy_component" { + for_each = local.component_environment_pairs + + project_id = oci_devops_project.devops_project.id + display_name = each.value.environment == "staging" ? "${each.value.name}-release" : "${each.value.name}-${each.value.environment}-deploy" + description = each.value.environment == "staging" ? "Promotes ${each.value.name} through staging approval and production" : "Deploys the ${each.value.environment} ${each.value.name} Helm release from OCIR" + freeform_tags = { + application = each.value.application_name + component = each.value.name + } + + deploy_pipeline_parameters { + items { + name = "component_chart_version" + default_value = each.value.chart_version + description = "${each.value.name} chart version to deploy" + } + items { + name = "image_repository" + default_value = each.value.image_repository + description = "Image repository for ${each.value.name}" + } + items { + name = "image_tag" + default_value = "" + description = each.value.environment == "staging" ? "Release candidate image tag to deploy, for example 1.0.0-rc.1" : "Image tag for ${each.value.name}; set explicitly for manual deployments" + } + + items { + name = "source_repository_id" + default_value = oci_devops_repository.application_source[each.value.name].id + description = "OCI DevOps source repository to tag after production succeeds" + } + + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_pipeline" "application_bootstrap" { + for_each = local.applications_by_name + + project_id = oci_devops_project.devops_project.id + display_name = "${each.value.name}-bootstrap" + description = "Initializes the ${each.value.name} namespace and OCIR pull secret on noprod and prod" + freeform_tags = { + application = each.value.name + role = "application-bootstrap" + } + + deploy_pipeline_parameters { + items { + name = "registry_username" + default_value = null + description = "OCIR pull username, for example tenancy-namespace/user" + } + items { + name = "pull_password_secret_ocid" + default_value = null + description = "Vault secret OCID containing the OCIR pull password or auth token" + } + items { + name = "secret_name" + default_value = var.namespace_init_secret_name + description = "Kubernetes docker-registry secret name" + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "application_bootstrap_namespace" { + for_each = local.application_bootstrap_targets + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.application_bootstrap_command_spec.id + deploy_pipeline_id = oci_devops_deploy_pipeline.application_bootstrap[each.value.application_name].id + deploy_stage_type = "SHELL" + description = "Create the ${each.value.namespace} namespace and OCIR pull secret on ${each.value.cluster_name}" + display_name = "${each.value.application_name}-${each.value.cluster_name}-bootstrap" + timeout_in_seconds = 600 + freeform_tags = { + application = each.value.application_name + cluster = each.value.cluster_name + namespace = each.value.namespace + role = "namespace-bootstrap" + } + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 1 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = each.value.subnet_id + nsg_ids = each.value.nsg_ids + } + } + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_pipeline.application_bootstrap[each.value.application_name].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "deploy_application" { + for_each = local.applications_by_name + + are_hooks_enabled = true + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_application[each.key].id + deploy_stage_type = "OKE_HELM_CHART_DEPLOYMENT" + description = "Install or upgrade the ${local.baseline_cluster_name} ${each.value.name} baseline Helm chart from OCIR" + display_name = "${each.value.name}-${local.baseline_cluster_name}-deploy" + helm_chart_deploy_artifact_id = oci_devops_deploy_artifact.application_chart[each.key].id + max_history = 5 + namespace = each.value.namespace + oke_cluster_deploy_environment_id = local.oke_environment_id + purpose = "EXECUTE_HELM_UPGRADE" + release_name = "${each.value.name}-${local.baseline_cluster_name}" + should_skip_crds = false + timeout_in_seconds = 600 + values_artifact_ids = [oci_devops_deploy_artifact.application_baseline_values[each.key].id] + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_pipeline.deploy_application[each.key].id + } + } + + rollback_policy { + policy_type = "AUTOMATED_STAGE_ROLLBACK_POLICY" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "approve_application_prod_baseline" { + for_each = local.applications_by_name + + approval_policy { + approval_policy_type = "COUNT_BASED_APPROVAL" + number_of_approvals_required = 1 + } + + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_application[each.key].id + deploy_stage_type = "MANUAL_APPROVAL" + description = "Approve promotion of ${each.value.name} namespace baseline from noprod to production" + display_name = "${each.value.name}-prod-approve" + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.deploy_application[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "deploy_application_prod" { + for_each = local.applications_by_name + + are_hooks_enabled = true + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_application[each.key].id + deploy_stage_type = "OKE_HELM_CHART_DEPLOYMENT" + description = "Install or upgrade the prod ${each.value.name} baseline Helm chart from OCIR" + display_name = "${each.value.name}-prod-deploy" + helm_chart_deploy_artifact_id = oci_devops_deploy_artifact.application_chart[each.key].id + max_history = 5 + namespace = each.value.prod_namespace + oke_cluster_deploy_environment_id = local.prod_oke_environment_id + purpose = "EXECUTE_HELM_UPGRADE" + release_name = each.value.name + should_skip_crds = false + timeout_in_seconds = 600 + values_artifact_ids = [oci_devops_deploy_artifact.application_prod_values[each.key].id] + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.approve_application_prod_baseline[each.key].id + } + } + + rollback_policy { + policy_type = "AUTOMATED_STAGE_ROLLBACK_POLICY" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "deploy_component" { + for_each = local.component_environment_pairs + + are_hooks_enabled = true + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component[each.key].id + deploy_stage_type = "OKE_HELM_CHART_DEPLOYMENT" + description = "Install or upgrade the ${each.value.environment} ${each.value.name} Helm chart from OCIR" + display_name = "${each.value.name}-${each.value.environment}-deploy" + helm_chart_deploy_artifact_id = oci_devops_deploy_artifact.component_chart[each.value.name].id + max_history = 5 + namespace = each.value.application_namespace + oke_cluster_deploy_environment_id = local.oke_environment_id + purpose = "EXECUTE_HELM_UPGRADE" + release_name = "${each.value.name}-${each.value.environment}" + should_skip_crds = false + timeout_in_seconds = 600 + values_artifact_ids = [oci_devops_deploy_artifact.component_values[each.key].id] + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_pipeline.deploy_component[each.key].id + } + } + + rollback_policy { + policy_type = "AUTOMATED_STAGE_ROLLBACK_POLICY" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "approve_component_prod_release" { + for_each = local.components_by_name + + approval_policy { + approval_policy_type = "COUNT_BASED_APPROVAL" + number_of_approvals_required = 1 + } + + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + deploy_stage_type = "MANUAL_APPROVAL" + description = "Approve promotion of ${each.value.name} from staging to production" + display_name = "${each.value.name}-prod-approve" + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.deploy_component["${each.key}:staging"].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "promote_component_release_image" { + for_each = local.components_by_name + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.promote_release_image_command_spec.id + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + deploy_stage_type = "SHELL" + description = "Retag the approved RC image as the final release image" + display_name = "${each.value.name}-release-image-promote" + timeout_in_seconds = 600 + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 1 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + } + } + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.approve_component_prod_release[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "deploy_component_prod" { + for_each = local.components_by_name + + are_hooks_enabled = true + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + deploy_stage_type = "OKE_HELM_CHART_DEPLOYMENT" + description = "Install or upgrade the prod ${each.value.name} Helm chart from OCIR" + display_name = "${each.value.name}-prod-deploy" + helm_chart_deploy_artifact_id = oci_devops_deploy_artifact.component_chart[each.key].id + max_history = 5 + namespace = each.value.application_prod_namespace + oke_cluster_deploy_environment_id = local.prod_oke_environment_id + purpose = "EXECUTE_HELM_UPGRADE" + release_name = each.value.name + should_skip_crds = false + timeout_in_seconds = 600 + values_artifact_ids = [oci_devops_deploy_artifact.component_prod_values[each.key].id] + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.promote_component_release_image[each.key].id + } + } + + rollback_policy { + policy_type = "AUTOMATED_STAGE_ROLLBACK_POLICY" + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "verify_component_prod" { + for_each = local.components_by_name + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.component_verify_deployment_command_spec.id + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + deploy_stage_type = "SHELL" + description = "Report the completed production Helm release status" + display_name = "${each.value.name}-prod-status" + timeout_in_seconds = 600 + freeform_tags = { + application = each.value.application_name + component = each.value.name + environment = "prod" + namespace = each.value.application_prod_namespace + release = each.value.name + role = "deployment-status" + } + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 1 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + } + } + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.tag_component_release_commit[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_deploy_stage" "tag_component_release_commit" { + for_each = local.components_by_name + + command_spec_deploy_artifact_id = oci_devops_deploy_artifact.tag_release_commit_command_spec.id + deploy_pipeline_id = oci_devops_deploy_pipeline.deploy_component["${each.key}:staging"].id + deploy_stage_type = "SHELL" + description = "Create the final source Git tag after production deployment succeeds" + display_name = "${each.value.name}-release-commit-tag" + timeout_in_seconds = 600 + + container_config { + container_config_type = "CONTAINER_INSTANCE_CONFIG" + compartment_id = var.compartment_id + shape_name = "CI.Standard.E4.Flex" + + shape_config { + ocpus = 1 + memory_in_gbs = 1 + } + + network_channel { + network_channel_type = "SERVICE_VNIC_CHANNEL" + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + } + } + + deploy_stage_predecessor_collection { + items { + id = oci_devops_deploy_stage.deploy_component_prod[each.key].id + } + } + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/environment.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/environment.tf new file mode 100644 index 000000000..84fac4141 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/environment.tf @@ -0,0 +1,41 @@ +resource "oci_devops_deploy_environment" "oke_environment" { + deploy_environment_type = "OKE_CLUSTER" + project_id = oci_devops_project.devops_project.id + cluster_id = var.oke_cluster_id + display_name = data.oci_containerengine_cluster.oke_cluster.name + description = "Pre-prod OKE cluster for Helm delivery" + + network_channel { + network_channel_type = "PRIVATE_ENDPOINT_CHANNEL" + subnet_id = var.oke_worker_subnet_id + nsg_ids = local.oke_worker_nsg_ids + } + + lifecycle { + ignore_changes = [ + display_name, + description + ] + } +} + +resource "oci_devops_deploy_environment" "prod_oke_environment" { + deploy_environment_type = "OKE_CLUSTER" + project_id = oci_devops_project.devops_project.id + cluster_id = local.prod_oke_cluster_id + display_name = data.oci_containerengine_cluster.prod_oke_cluster.name + description = "Production OKE cluster for release deployments" + + network_channel { + network_channel_type = "PRIVATE_ENDPOINT_CHANNEL" + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + } + + lifecycle { + ignore_changes = [ + display_name, + description + ] + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/locals.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/locals.tf new file mode 100644 index 000000000..06ef65a96 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/locals.tf @@ -0,0 +1,163 @@ +locals { + region_key = lower([for s in data.oci_identity_region_subscriptions.region_subscriptions.region_subscriptions : s if s.region_name == var.region][0].region_key) + namespace = data.oci_artifacts_container_configuration.ocir_config.namespace + + platform_repo_path = "repos/pipelines" + project_repo_prefix = replace(lower(var.devops_project_name), "/[^a-z0-9._-]+/", "-") + + applications_by_name = { + for application in var.applications : application.name => { + name = application.name + chart_repository_name = coalesce(application.chart_repository_name, "${application.name}-chart") + chart_path = coalesce(application.chart_path, application.name) + chart_version = application.chart_version + namespace = coalesce(application.namespace, application.name) + prod_namespace = coalesce(application.prod_namespace, coalesce(application.namespace, application.name)) + kubernetes_group = application.kubernetes_group + repo_path = "repos/generated/charts/${application.name}" + chart_repo_prefix = "${local.project_repo_prefix}/charts" + ocir_chart = "oci://${local.region_key}.ocir.io/${local.namespace}/${local.project_repo_prefix}/charts/${application.name}" + components = application.components + } + } + + components_by_name = merge([ + for application in var.applications : { + for component in application.components : component.name => { + name = component.name + application_name = application.name + application_namespace = coalesce(application.namespace, application.name) + application_prod_namespace = coalesce(application.prod_namespace, coalesce(application.namespace, application.name)) + application_chart_path = coalesce(application.chart_path, application.name) + chart_version = component.chart_version + build_spec_path = coalesce(component.build_spec_path, "${component.name}-build-pipeline.yaml") + generate_build_spec = component.build_spec_path == null + repo_path = "repos/generated/components/${component.name}" + chart_path = "${coalesce(application.chart_path, application.name)}/charts/${component.name}" + chart_repo_prefix = "${local.project_repo_prefix}/charts/${application.name}" + image_repo_prefix = "${local.project_repo_prefix}/${application.name}/${component.name}" + ocir_chart = "oci://${local.region_key}.ocir.io/${local.namespace}/${local.project_repo_prefix}/charts/${application.name}/${component.name}" + image_repository = "${local.region_key}.ocir.io/${local.namespace}/${local.project_repo_prefix}/${application.name}/${component.name}" + } + } + ]...) + + generated_component_build_specs = { + for name, component in local.components_by_name : name => component if component.generate_build_spec + } + + custom_build_spec_paths = distinct([ + for component in values(local.components_by_name) : component.build_spec_path + if !component.generate_build_spec + ]) + + custom_build_specs_by_path = { + for path in local.custom_build_spec_paths : path => { + component = local.components_by_name[sort([ + for name, component in local.components_by_name : name + if !component.generate_build_spec && component.build_spec_path == path + ])[0]] + component_names = sort([ + for name, component in local.components_by_name : name + if !component.generate_build_spec && component.build_spec_path == path + ]) + } + } + + component_environment_pairs = merge([ + for component_name, component in local.components_by_name : { + for environment in ["dev", "staging"] : "${component_name}:${environment}" => merge(component, { + environment = environment + }) + } + ]...) + + application_bootstrap_targets = merge([ + for application_name, application in local.applications_by_name : { + "${application_name}:noprod" = { + application_name = application_name + cluster_name = "noprod" + namespace = application.namespace + subnet_id = var.oke_worker_subnet_id + nsg_ids = local.oke_worker_nsg_ids + } + "${application_name}:prod" = { + application_name = application_name + cluster_name = "prod" + namespace = application.prod_namespace + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + } + } + ]...) + + default_application_environment = "dev" + baseline_cluster_name = "noprod" + kube_endpoint = "PRIVATE_ENDPOINT" + + prod_oke_cluster_id = var.prod_oke_cluster_id + prod_oke_worker_subnet_id = var.prod_oke_worker_subnet_id + prod_oke_worker_nsg_id = var.prod_oke_worker_nsg_id + oke_worker_nsg_ids = try(trimspace(var.oke_worker_nsg_id), "") == "" ? [] : [var.oke_worker_nsg_id] + prod_oke_worker_nsg_ids = try(trimspace(local.prod_oke_worker_nsg_id), "") == "" ? [] : [local.prod_oke_worker_nsg_id] + + oke_environment_id = oci_devops_deploy_environment.oke_environment.id + prod_oke_environment_id = oci_devops_deploy_environment.prod_oke_environment.id + + cluster_admin_repo_path = "repos/generated/cluster-admin" + cluster_admin_singleton = var.enable_cluster_admin ? { enabled = true } : {} + cluster_admin_tags = { + owner = "cluster-administrators" + purpose = "cluster-administration" + scope = "operations" + } + # A shared topology keeps both physical-cluster pipelines structurally identical. + # The nested noprod list is retained as a fallback for existing stack inputs. + cluster_admin_tools = var.enable_cluster_admin ? coalesce( + try(var.cluster_administration.tools, null), + try(var.cluster_administration.noprod.tools, null), + [] + ) : [] + cluster_admin_clusters = var.enable_cluster_admin ? { + noprod = { + name = "noprod" + approval_required = false + cluster_id = var.oke_cluster_id + environment_id = local.oke_environment_id + subnet_id = var.oke_worker_subnet_id + nsg_ids = local.oke_worker_nsg_ids + tools = local.cluster_admin_tools + } + prod = { + name = "prod" + approval_required = true + cluster_id = local.prod_oke_cluster_id + environment_id = local.prod_oke_environment_id + subnet_id = local.prod_oke_worker_subnet_id + nsg_ids = local.prod_oke_worker_nsg_ids + tools = local.cluster_admin_tools + } + } : {} + + cluster_admin_tool_pairs = merge([ + for cluster_name, cluster in local.cluster_admin_clusters : { + for tool in cluster.tools : "${cluster_name}:${tool.name}" => { + cluster_name = cluster_name + tool_name = tool.name + namespace = coalesce(tool.namespace, tool.name) + dependencies = try(tool.depends_on, []) + approval_required = cluster.approval_required + cluster_id = cluster.cluster_id + environment_id = cluster.environment_id + subnet_id = cluster.subnet_id + nsg_ids = cluster.nsg_ids + values_path = "cluster-admin/${cluster_name}/tools/${tool.name}/values.yaml" + } + } + ]...) + + cluster_admin_approval_clusters = { + for name, cluster in local.cluster_admin_clusters : name => cluster if cluster.approval_required + } + +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/outputs.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/outputs.tf new file mode 100644 index 000000000..3d0dd7916 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/outputs.tf @@ -0,0 +1,153 @@ +output "devops_project_id" { + description = "OCID of the OCI DevOps project." + value = oci_devops_project.devops_project.id +} + +output "pipelines_repository_url" { + description = "HTTPS URL of the shared pipelines repository." + value = oci_devops_repository.platform_pipelines.http_url +} + +output "cluster_admin_repository_url" { + description = "HTTPS URL of the cluster administration repository." + value = try(oci_devops_repository.cluster_admin["enabled"].http_url, null) +} + +output "cluster_admin_values_repository_id" { + description = "OCID of the Generic Artifact repository containing immutable cluster tool values and deployment plans." + value = try(oci_artifacts_repository.cluster_admin_values["enabled"].id, null) +} + +output "cluster_admin_build_pipeline_ids" { + description = "Shared cluster administration build pipeline OCIDs." + value = var.enable_cluster_admin ? { + build = oci_devops_build_pipeline.cluster_admin_build["enabled"].id + mirror = oci_devops_build_pipeline.cluster_admin_mirror["enabled"].id + pr = oci_devops_build_pipeline.cluster_admin_pr["enabled"].id + } : {} +} + +output "cluster_admin_deploy_pipeline_ids" { + description = "Cluster administration deployment and decommission pipeline OCIDs." + value = merge( + { for name, pipeline in oci_devops_deploy_pipeline.cluster_admin : name => pipeline.id }, + { for name, pipeline in oci_devops_deploy_pipeline.cluster_admin_decommission : "${name}-decommission" => pipeline.id } + ) +} + +output "component_source_repository_urls" { + description = "Component name to OCI DevOps source repository URL." + value = { + for name, repo in oci_devops_repository.application_source : name => repo.http_url + } +} + +output "application_chart_repository_urls" { + description = "Application name to OCI DevOps chart repository URL." + value = { + for name, repo in oci_devops_repository.application_chart : name => repo.http_url + } +} + +output "component_build_pipeline_ids" { + description = "Component name to build pipeline OCID." + value = { + for name, pipeline in oci_devops_build_pipeline.application_delivery : name => pipeline.id + } +} + +output "component_pr_pipeline_ids" { + description = "Component name to pull request validation pipeline OCID." + value = { + for name, pipeline in oci_devops_build_pipeline.application_pull_request : name => pipeline.id + } +} + +output "component_release_build_pipeline_ids" { + description = "Component name to release build pipeline OCID." + value = { + for name, pipeline in oci_devops_build_pipeline.release_application : name => pipeline.id + } +} + +output "application_package_pipeline_ids" { + description = "Application name to baseline chart package pipeline OCID." + value = { + for name, pipeline in oci_devops_build_pipeline.application_baseline_package : name => pipeline.id + } +} + +output "application_deploy_pipeline_ids" { + description = "Application name to baseline deployment pipeline OCID." + value = { + for name, pipeline in oci_devops_deploy_pipeline.deploy_application : name => pipeline.id + } +} + +output "application_bootstrap_pipeline_ids" { + description = "Application name to parallel noprod/prod namespace bootstrap deployment pipeline OCID." + value = { + for name, pipeline in oci_devops_deploy_pipeline.application_bootstrap : name => pipeline.id + } +} + +output "component_deploy_pipeline_ids" { + description = "Component and environment to deployment pipeline OCID." + value = { + for key, pipeline in oci_devops_deploy_pipeline.deploy_component : key => pipeline.id + } +} + +output "oke_noprod_deploy_environment_id" { + description = "OCID of the pre-production OKE deployment environment." + value = local.oke_environment_id +} + +output "oke_prod_deploy_environment_id" { + description = "OCID of the production OKE deployment environment." + value = local.prod_oke_environment_id +} + +output "application_components" { + description = "Application name to component names." + value = { + for name, application in local.applications_by_name : name => [ + for component in application.components : component.name + ] + } +} + +output "application_namespaces" { + description = "Application name to pre-production Kubernetes namespace." + value = { + for name, application in local.applications_by_name : name => application.namespace + } +} + +output "application_prod_namespaces" { + description = "Application name to production Kubernetes namespace." + value = { + for name, application in local.applications_by_name : name => application.prod_namespace + } +} + +output "application_ocir_charts" { + description = "Application name to umbrella Helm chart OCI URL." + value = { + for name, application in local.applications_by_name : name => application.ocir_chart + } +} + +output "component_ocir_charts" { + description = "Component name to Helm chart OCI URL." + value = { + for name, component in local.components_by_name : name => component.ocir_chart + } +} + +output "component_image_repositories" { + description = "Component name to OCIR image repository." + value = { + for name, component in local.components_by_name : name => component.image_repository + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/project.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/project.tf new file mode 100644 index 000000000..dce6eb117 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/project.tf @@ -0,0 +1,50 @@ +locals { + devops_log_name = try(trimspace(var.devops_log_name), "") == "" ? "${var.devops_log_group_name}_${var.devops_project_name}_log" : var.devops_log_name +} + +resource "oci_ons_notification_topic" "devops_notification_topic" { + compartment_id = var.compartment_id + name = var.notification_topic_name + description = var.notification_topic_description + count = var.create_notification_topic ? 1 : 0 +} + +resource "oci_devops_project" "devops_project" { + compartment_id = var.compartment_id + name = var.devops_project_name + description = var.devops_project_description + + notification_config { + topic_id = var.create_notification_topic ? oci_ons_notification_topic.devops_notification_topic[0].id : var.notification_topic_id + } +} + +resource "oci_logging_log_group" "devops_log_group" { + count = var.devops_log_is_enabled ? 1 : 0 + + compartment_id = var.compartment_id + display_name = var.devops_log_group_name + description = var.devops_log_group_description +} + +resource "oci_logging_log" "devops_log" { + count = var.devops_log_is_enabled ? 1 : 0 + + display_name = local.devops_log_name + log_group_id = oci_logging_log_group.devops_log_group[0].id + log_type = "SERVICE" + + configuration { + source { + category = "all" + resource = oci_devops_project.devops_project.id + service = "devops" + source_type = "OCISERVICE" + } + + compartment_id = var.compartment_id + } + + is_enabled = true + retention_duration = var.devops_log_retention_period_in_days +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/protected_branches.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/protected_branches.tf new file mode 100644 index 000000000..155ae44ee --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/protected_branches.tf @@ -0,0 +1,57 @@ +resource "oci_devops_repository_protected_branch_management" "application_source_main" { + for_each = local.components_by_name + + repository_id = oci_devops_repository.application_source[each.key].id + branch_name = "main" + + protection_levels = [ + "PULL_REQUEST_MERGE_ONLY", + ] + + depends_on = [ + null_resource.seed_application_source + ] + + lifecycle { + # Provider normalization otherwise replaces branch protection on every plan. + ignore_changes = all + } +} + +resource "oci_devops_repository_protected_branch_management" "application_chart_main" { + for_each = local.applications_by_name + + repository_id = oci_devops_repository.application_chart[each.key].id + branch_name = "main" + + protection_levels = [ + "PULL_REQUEST_MERGE_ONLY", + ] + + depends_on = [ + null_resource.seed_application_chart_components + ] + + lifecycle { + # Provider normalization otherwise replaces branch protection on every plan. + ignore_changes = all + } +} + +resource "oci_devops_repository_protected_branch_management" "cluster_admin_main" { + for_each = local.cluster_admin_singleton + + repository_id = oci_devops_repository.cluster_admin[each.key].id + branch_name = "main" + + protection_levels = [ + "PULL_REQUEST_MERGE_ONLY", + ] + + depends_on = [null_resource.seed_cluster_admin_entities] + + lifecycle { + # Provider normalization otherwise replaces branch protection on every plan. + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/provider.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/provider.tf new file mode 100644 index 000000000..cbadb915d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/provider.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + local = { + source = "hashicorp/local" + } + null = { + source = "hashicorp/null" + } + oci = { + source = "oracle/oci" + } + } +} + diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repositories.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repositories.tf new file mode 100644 index 000000000..415e684f8 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repositories.tf @@ -0,0 +1,461 @@ +resource "oci_devops_repository" "platform_pipelines" { + name = "pipelines" + project_id = oci_devops_project.devops_project.id + description = "Reusable pipeline assets for application delivery" + repository_type = "HOSTED" + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_repository" "application_source" { + for_each = local.components_by_name + + name = each.value.name + project_id = oci_devops_project.devops_project.id + description = "Source code repository for the ${each.value.name} component. Helm chart specifications are intentionally kept separate." + repository_type = "HOSTED" + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_repository" "application_chart" { + for_each = local.applications_by_name + + name = each.value.chart_repository_name + project_id = oci_devops_project.devops_project.id + description = "Helm chart repository for the ${each.value.name} application. Umbrella and component subchart paths follow separate package and deploy lifecycles." + repository_type = "HOSTED" + + lifecycle { + ignore_changes = all + } +} + +resource "local_file" "application_source_env" { + for_each = local.components_by_name + + filename = "${path.root}/${each.value.repo_path}/.oci-devops/application.env" + content = templatefile("${path.root}/templates/application-source-env.tpl", { + component_name = each.value.name + }) +} + +resource "local_file" "application_source_pull_request_pipeline" { + for_each = local.components_by_name + + filename = "${path.root}/${each.value.repo_path}/.oci-devops/pull-request-pipeline.yaml" + content = <<-EOT + version: 0.1 + component: build + timeoutInSeconds: 1000 + runAs: root + shell: bash + + steps: + - type: Command + name: Run component tests + failImmediatelyOnError: true + command: | + echo "Unit tests succeeded" + EOT +} + +resource "local_file" "application_source_readme" { + for_each = local.components_by_name + + filename = "${path.root}/${each.value.repo_path}/README.md" + content = templatefile("${path.root}/templates/application-source-README.md.tpl", { + application_name = each.value.application_name + chart_repository_name = local.applications_by_name[each.value.application_name].chart_repository_name + component_name = each.value.name + devops_project_prefix = local.project_repo_prefix + }) +} + +resource "local_file" "application_source_dockerfile" { + for_each = local.components_by_name + + filename = "${path.root}/${each.value.repo_path}/Dockerfile" + content = templatefile("${path.root}/templates/application-source-dockerfile.tpl", { + component_name = each.value.name + }) +} + +resource "local_file" "application_chart_readme" { + for_each = local.applications_by_name + + filename = "${path.root}/${each.value.repo_path}/README.md" + content = templatefile("${path.root}/templates/application-chart-README.md.tpl", { + application_name = each.value.name + application_path = each.value.chart_path + chart_repository_name = each.value.chart_repository_name + component_name = join(", ", [for component in each.value.components : component.name]) + }) +} + +resource "local_file" "application_chart_chart_yaml" { + for_each = local.applications_by_name + + filename = "${path.root}/${each.value.repo_path}/${each.value.chart_path}/Chart.yaml" + content = templatefile("${path.root}/templates/application-chart-Chart.yaml.tpl", { + application_name = each.value.name + chart_version = each.value.chart_version + }) +} + +resource "local_file" "application_chart_values_yaml" { + for_each = local.applications_by_name + + filename = "${path.root}/${each.value.repo_path}/${each.value.chart_path}/values.yaml" + content = templatefile("${path.root}/templates/application-chart-values.yaml.tpl", {}) +} + +resource "local_file" "application_chart_helmignore" { + for_each = local.applications_by_name + + filename = "${path.root}/${each.value.repo_path}/${each.value.chart_path}/.helmignore" + content = templatefile("${path.root}/templates/application-chart-helmignore.tpl", {}) +} + +resource "local_file" "component_chart_chart_yaml" { + for_each = local.components_by_name + + filename = "${path.root}/${local.applications_by_name[each.value.application_name].repo_path}/${each.value.chart_path}/Chart.yaml" + content = templatefile("${path.root}/templates/component-chart-Chart.yaml.tpl", { + chart_version = each.value.chart_version + component_name = each.value.name + }) +} + +resource "local_file" "component_chart_values_yaml" { + for_each = local.components_by_name + + filename = "${path.root}/${local.applications_by_name[each.value.application_name].repo_path}/${each.value.chart_path}/values.yaml" + content = templatefile("${path.root}/templates/component-chart-values.yaml.tpl", { + application_name = each.value.application_name + component_name = each.value.name + }) +} + +resource "local_file" "component_chart_deployment_yaml" { + for_each = local.components_by_name + + filename = "${path.root}/${local.applications_by_name[each.value.application_name].repo_path}/${each.value.chart_path}/templates/deployment.yaml" + content = templatefile("${path.root}/templates/component-chart-deployment.yaml.tpl", {}) +} + +resource "local_file" "component_chart_service_yaml" { + for_each = local.components_by_name + + filename = "${path.root}/${local.applications_by_name[each.value.application_name].repo_path}/${each.value.chart_path}/templates/service.yaml" + content = templatefile("${path.root}/templates/component-chart-service.yaml.tpl", {}) +} + +resource "local_file" "component_chart_serviceaccount_yaml" { + for_each = local.components_by_name + + filename = "${path.root}/${local.applications_by_name[each.value.application_name].repo_path}/${each.value.chart_path}/templates/serviceaccount.yaml" + content = templatefile("${path.root}/templates/component-chart-serviceaccount.yaml.tpl", {}) +} + +resource "local_file" "application_delivery_pipeline" { + for_each = local.generated_component_build_specs + + filename = "${path.root}/${local.platform_repo_path}/${each.value.build_spec_path}" + content = templatefile("${path.root}/templates/application-delivery-pipeline.yaml.tpl", { + application_chart_path = each.value.application_chart_path + component_chart_repo_prefix = each.value.chart_repo_prefix + component_image_repo_prefix = each.value.image_repo_prefix + component_chart_path = each.value.chart_path + component_image_repository = each.value.image_repository + component_name = each.value.name + region = var.region + region_key = local.region_key + repo_compartment_id = var.compartment_id + tenancy_namespace = local.namespace + }) +} + +resource "local_file" "custom_application_delivery_pipeline" { + for_each = local.custom_build_specs_by_path + + filename = "${path.root}/${local.platform_repo_path}/${each.key}" + content = join("\n", [ + "# Starter generated for components: ${join(", ", each.value.component_names)}.", + "# This explicit build_spec_path is user-owned after its first commit and is never refreshed by Resource Manager.", + templatefile("${path.root}/templates/application-delivery-pipeline.yaml.tpl", { + application_chart_path = each.value.component.application_chart_path + component_chart_repo_prefix = each.value.component.chart_repo_prefix + component_image_repo_prefix = each.value.component.image_repo_prefix + component_chart_path = each.value.component.chart_path + component_image_repository = each.value.component.image_repository + component_name = each.value.component.name + region = var.region + region_key = local.region_key + repo_compartment_id = var.compartment_id + tenancy_namespace = local.namespace + }) + ]) +} + +resource "local_file" "platform_pipelines_readme" { + filename = "${path.root}/${local.platform_repo_path}/README.md" + content = templatefile("${path.root}/templates/pipelines-README.md.tpl", { + application_name = join(", ", keys(local.applications_by_name)) + component_name = join(", ", keys(local.components_by_name)) + }) +} + +resource "local_file" "application_baseline_pipeline" { + for_each = local.applications_by_name + + filename = "${path.root}/${local.platform_repo_path}/${each.value.name}-package-pipeline.yaml" + content = templatefile("${path.root}/templates/application-baseline-pipeline.yaml.tpl", { + application_chart_name = each.value.name + application_chart_path = each.value.chart_path + application_chart_repo_prefix = each.value.chart_repo_prefix + region = var.region + region_key = local.region_key + repo_compartment_id = var.compartment_id + tenancy_namespace = local.namespace + }) +} + +resource "local_file" "helm_chart_pipeline" { + filename = "${path.root}/${local.platform_repo_path}/helm-chart-pipeline.yaml" + content = templatefile("${path.root}/templates/helm-chart-pipeline.yaml.tpl", { + region = var.region + region_key = local.region_key + repo_compartment_id = var.compartment_id + tenancy_namespace = local.namespace + }) +} + +resource "local_file" "release_pipeline" { + for_each = local.components_by_name + + filename = "${path.root}/${local.platform_repo_path}/${each.value.name}-release-pipeline.yaml" + content = templatefile("${path.root}/templates/release-pipeline.yaml.tpl", { + application_chart_path = each.value.chart_path + component_chart_repo_prefix = each.value.chart_repo_prefix + component_image_repo_prefix = each.value.image_repo_prefix + component_image_repository = each.value.image_repository + region = var.region + region_key = local.region_key + repo_compartment_id = var.compartment_id + tenancy_namespace = local.namespace + }) +} + +resource "null_resource" "seed_platform_shared" { + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.platform_pipelines.http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + # OCI can create a hosted repository with an empty initial commit. Add-only + # seeds missing starter paths without overwriting adopted repository content. + SEED_MODE = "add-only" + SEED_PATHS = join("\n", ["README.md", "helm-chart-pipeline.yaml", "script"]) + SOURCE_REPO = "/${local.platform_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.platform_pipelines.id + seed_mode = "add-only" + } + + depends_on = [ + local_file.helm_chart_pipeline, + local_file.platform_pipelines_readme + ] +} + +resource "null_resource" "seed_platform_entities" { + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.platform_pipelines.http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", concat( + [for name in sort(keys(local.applications_by_name)) : "${name}-package-pipeline.yaml"], + [for name in sort(keys(local.generated_component_build_specs)) : local.generated_component_build_specs[name].build_spec_path], + [for name in sort(keys(local.components_by_name)) : "${name}-release-pipeline.yaml"], + sort(local.custom_build_spec_paths) + )) + SOURCE_REPO = "/${local.platform_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.platform_pipelines.id + application_names = join(",", sort(keys(local.applications_by_name))) + component_names = join(",", sort(keys(local.components_by_name))) + generated_build_spec_paths = join(",", sort([for component in values(local.generated_component_build_specs) : component.build_spec_path])) + custom_build_spec_paths = join(",", sort(local.custom_build_spec_paths)) + } + + depends_on = [ + null_resource.seed_platform_shared, + local_file.application_baseline_pipeline, + local_file.application_delivery_pipeline, + local_file.custom_application_delivery_pipeline, + local_file.release_pipeline + ] +} + +resource "null_resource" "refresh_platform_development" { + count = var.development_mode ? 1 : 0 + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.platform_pipelines.http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "refresh" + SEED_PATHS = join("\n", concat( + ["README.md", "helm-chart-pipeline.yaml", "script"], + [for name in sort(keys(local.applications_by_name)) : "${name}-package-pipeline.yaml"], + [for name in sort(keys(local.generated_component_build_specs)) : local.generated_component_build_specs[name].build_spec_path], + [for name in sort(keys(local.components_by_name)) : "${name}-release-pipeline.yaml"] + )) + SOURCE_REPO = "/${local.platform_repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.platform_pipelines.id + content_hash = sha256(join("", concat( + [ + local_file.platform_pipelines_readme.content, + local_file.helm_chart_pipeline.content, + ], + [for name in sort(keys(local.applications_by_name)) : local_file.application_baseline_pipeline[name].content], + [for name in sort(keys(local.generated_component_build_specs)) : local_file.application_delivery_pipeline[name].content], + [for name in sort(keys(local.components_by_name)) : local_file.release_pipeline[name].content], + [ + for script_path in sort(fileset("${path.root}/${local.platform_repo_path}/script", "**")) : + filesha256("${path.root}/${local.platform_repo_path}/script/${script_path}") + ] + ))) + } + + depends_on = [null_resource.seed_platform_entities] +} + +resource "null_resource" "seed_application_source" { + for_each = local.components_by_name + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.application_source[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", ["README.md", "Dockerfile", ".oci-devops"]) + SOURCE_REPO = "/${each.value.repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.application_source[each.key].id + component_name = each.key + seed_mode = "add-only" + } + + depends_on = [ + local_file.application_source_dockerfile, + local_file.application_source_env, + local_file.application_source_pull_request_pipeline, + local_file.application_source_readme + ] +} + +resource "null_resource" "seed_application_chart_baseline" { + for_each = local.applications_by_name + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.application_chart[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", [ + "README.md", + "${each.value.chart_path}/Chart.yaml", + "${each.value.chart_path}/values.yaml", + "${each.value.chart_path}/.helmignore" + ]) + SOURCE_REPO = "/${each.value.repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.application_chart[each.key].id + application_name = each.key + seed_mode = "add-only" + } + + depends_on = [ + local_file.application_chart_readme, + local_file.application_chart_chart_yaml, + local_file.application_chart_values_yaml, + local_file.application_chart_helmignore + ] +} + +resource "null_resource" "seed_application_chart_components" { + for_each = local.applications_by_name + + provisioner "local-exec" { + command = "chmod +x ./script/seed_repo.sh && ./script/seed_repo.sh" + environment = { + REPO_CLONE_URL = oci_devops_repository.application_chart[each.key].http_url + GIT_USERNAME = var.git_username + GIT_PASSWORD = var.git_password + REGION = var.region + SEED_MODE = "add-only" + SEED_PATHS = join("\n", [ + for component in each.value.components : "${each.value.chart_path}/charts/${component.name}" + ]) + SOURCE_REPO = "/${each.value.repo_path}" + } + working_dir = path.root + } + + triggers = { + repo_id = oci_devops_repository.application_chart[each.key].id + application_name = each.key + component_names = join(",", sort([for component in each.value.components : component.name])) + } + + depends_on = [ + null_resource.seed_application_chart_baseline, + local_file.component_chart_chart_yaml, + local_file.component_chart_values_yaml, + local_file.component_chart_deployment_yaml, + local_file.component_chart_service_yaml, + local_file.component_chart_serviceaccount_yaml + ] +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repository_settings.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repository_settings.tf new file mode 100644 index 000000000..2acd977a6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/repository_settings.tf @@ -0,0 +1,44 @@ +resource "oci_devops_project_repository_setting" "project" { + project_id = oci_devops_project.devops_project.id + + merge_settings { + allowed_merge_strategies = [ + "MERGE_COMMIT", + "SQUASH", + "SQUASH_FAST_FORWARD_ONLY", + ] + default_merge_strategy = "SQUASH" + } + + lifecycle { + # OCI normalizes merge settings after creation; preserve the adopted settings. + ignore_changes = all + } +} + +resource "oci_devops_repository_setting" "repositories" { + for_each = merge( + { + pipelines = oci_devops_repository.platform_pipelines.id + }, + { for key, repo in oci_devops_repository.cluster_admin : "cluster-admin" => repo.id }, + { for name, repo in oci_devops_repository.application_chart : "chart:${name}" => repo.id }, + { for name, repo in oci_devops_repository.application_source : "source:${name}" => repo.id } + ) + + repository_id = each.value + + merge_settings { + allowed_merge_strategies = [ + "MERGE_COMMIT", + "SQUASH", + "SQUASH_FAST_FORWARD_ONLY", + ] + default_merge_strategy = "SQUASH" + } + + lifecycle { + # OCI normalizes merge settings after creation; preserve the adopted settings. + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/triggers.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/triggers.tf new file mode 100644 index 000000000..4165cde07 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/triggers.tf @@ -0,0 +1,165 @@ +resource "oci_devops_trigger" "application_source_commit" { + for_each = local.components_by_name + + display_name = "${each.value.name}-main-commit" + description = "Runs ${each.value.name} delivery when the source repository changes" + project_id = oci_devops_project.devops_project.id + trigger_source = "DEVOPS_CODE_REPOSITORY" + + actions { + build_pipeline_id = oci_devops_build_pipeline.application_delivery[each.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PUSH"] + + include { + head_ref = "main" + } + } + } + + actions { + build_pipeline_id = oci_devops_build_pipeline.application_pull_request[each.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PULL_REQUEST_CREATED", "PULL_REQUEST_UPDATED"] + + include { + base_ref = "main" + } + } + } + + repository_id = oci_devops_repository.application_source[each.key].id + + depends_on = [ + oci_devops_build_pipeline_stage.trigger_dev_deployment, + oci_devops_build_pipeline_stage.application_pull_request + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_trigger" "application_chart_commit" { + for_each = local.applications_by_name + + display_name = "${each.value.name}-chart-main-commit" + description = "Routes chart repository changes to component or application baseline lifecycles" + project_id = oci_devops_project.devops_project.id + trigger_source = "DEVOPS_CODE_REPOSITORY" + + dynamic "actions" { + for_each = { + for component in each.value.components : component.name => local.components_by_name[component.name] + } + + content { + build_pipeline_id = oci_devops_build_pipeline.application_delivery[actions.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PUSH"] + + include { + head_ref = "main" + + file_filter { + file_paths = ["${actions.value.chart_path}/**"] + } + } + } + } + } + + actions { + build_pipeline_id = oci_devops_build_pipeline.application_baseline_package[each.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PUSH"] + + include { + head_ref = "main" + + file_filter { + file_paths = ["${each.value.chart_path}/**"] + } + } + + exclude { + file_filter { + file_paths = ["${each.value.chart_path}/charts/**"] + } + } + } + } + + repository_id = oci_devops_repository.application_chart[each.key].id + + depends_on = [ + oci_devops_build_pipeline_stage.trigger_application_baseline_deployment, + oci_devops_build_pipeline_stage.trigger_dev_deployment + ] + + lifecycle { + ignore_changes = all + } +} + +resource "oci_devops_trigger" "cluster_admin_commit" { + for_each = local.cluster_admin_singleton + + display_name = "cluster-admin-main-commit" + description = "Validates and deploys changed cluster administration targets" + freeform_tags = merge(local.cluster_admin_tags, { + role = "source-trigger" + }) + project_id = oci_devops_project.devops_project.id + trigger_source = "DEVOPS_CODE_REPOSITORY" + repository_id = oci_devops_repository.cluster_admin[each.key].id + + actions { + build_pipeline_id = oci_devops_build_pipeline.cluster_admin_build[each.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PUSH"] + + include { + head_ref = "main" + } + } + } + + actions { + build_pipeline_id = oci_devops_build_pipeline.cluster_admin_pr[each.key].id + type = "TRIGGER_BUILD_PIPELINE" + + filter { + trigger_source = "DEVOPS_CODE_REPOSITORY" + events = ["PULL_REQUEST_CREATED", "PULL_REQUEST_UPDATED"] + + include { + base_ref = "main" + } + } + } + + depends_on = [ + oci_devops_build_pipeline_stage.cluster_admin_build, + oci_devops_build_pipeline_stage.cluster_admin_pr + ] + + lifecycle { + ignore_changes = all + } +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/variables.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/variables.tf new file mode 100644 index 000000000..5e41f0396 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/devops/variables.tf @@ -0,0 +1,110 @@ +variable "compartment_id" {} +variable "region" {} +variable "tenancy_id" {} +variable "git_username" {} +variable "git_password" { + sensitive = true +} + +variable "create_notification_topic" { + type = bool +} +variable "notification_topic_id" {} +variable "notification_topic_name" {} +variable "notification_topic_description" {} + +variable "devops_project_name" {} +variable "devops_project_description" {} +variable "devops_log_group_name" {} +variable "devops_log_group_description" {} +variable "devops_log_name" {} +variable "devops_log_is_enabled" { + type = bool +} +variable "devops_log_retention_period_in_days" { + type = number +} + +variable "oke_cluster_id" {} +variable "oke_worker_subnet_id" {} +variable "oke_worker_nsg_id" {} +variable "prod_oke_cluster_id" { + default = null +} +variable "prod_oke_compartment_id" { + default = null +} +variable "prod_oke_worker_subnet_id" { + default = null +} +variable "prod_oke_worker_nsg_id" { + default = null +} +variable "namespace_init_secret_name" {} + +variable "applications" { + type = list(object({ + name = string + chart_repository_name = optional(string) + chart_path = optional(string) + chart_version = optional(string, "0.1.0") + namespace = optional(string) + prod_namespace = optional(string) + kubernetes_group = optional(string, "") + components = list(object({ + name = string + chart_version = optional(string, "0.1.0") + build_spec_path = optional(string) + })) + })) +} + +variable "enable_cluster_admin" { + type = bool + default = false +} + +variable "cluster_admin_artifact_repository_name" { + type = string + default = "" +} + +variable "cluster_administration" { + type = object({ + tools = optional(list(object({ + name = string + repository = string + chart = string + version = string + namespace = optional(string) + depends_on = optional(list(string), []) + }))) + noprod = optional(object({ + approval_required = optional(bool, false) + tools = optional(list(object({ + name = string + repository = string + chart = string + version = string + namespace = optional(string) + depends_on = optional(list(string), []) + }))) + })) + prod = optional(object({ + approval_required = optional(bool, true) + tools = optional(list(object({ + name = string + repository = string + chart = string + version = string + namespace = optional(string) + depends_on = optional(list(string), []) + }))) + })) + }) +} + +variable "development_mode" { + type = bool + default = false +} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/iam/dynamic-groups.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/iam/dynamic-groups.tf new file mode 100644 index 000000000..309b77d1e --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/modules/iam/dynamic-groups.tf @@ -0,0 +1,28 @@ +data "oci_identity_domain" "devops_domain" { + domain_id = var.iam_domain_id +} + +locals { + idcs_endpoint = data.oci_identity_domain.devops_domain.url + + matching_rule = </ + Chart.yaml + values.yaml + .helmignore + charts/ + / + Chart.yaml + values.yaml + templates/ +``` + +## Lifecycles + +The umbrella chart and component charts have separate lifecycles: + +- `` is the umbrella baseline chart. It owns shared namespace resources. +- `/charts/` is the component chart. It owns the component `Deployment`, `Service`, and `ServiceAccount`. + +Trigger behavior: + +- Changes under `/charts//**` package only that component chart and redeploy dev. +- Changes under `/**`, excluding `charts/**`, package only the umbrella baseline chart and start `-deploy`. +- Source code changes are handled in the separate `` source repository. + +The umbrella chart has no component dependencies, and `.helmignore` excludes `charts/` from baseline loading and packaging. Component environments are deployed by installing each component chart directly with environment-specific values. + +Resource Manager adds missing application/component starter paths but never overwrites an existing repository path. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/.helmignore b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/.helmignore new file mode 100644 index 000000000..ee3892e87 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/.helmignore @@ -0,0 +1 @@ +charts/ diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/Chart.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/Chart.yaml new file mode 100644 index 000000000..a8ba9eacc --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: sample-app +description: Namespace umbrella chart for the sample-app application +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/Chart.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/Chart.yaml new file mode 100644 index 000000000..d683d52f5 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: sample-component +description: Sample application component workload subchart for the OKE Helm starter stack +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/deployment.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/deployment.yaml new file mode 100644 index 000000000..685af48fd --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.nameOverride }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + spec: + serviceAccountName: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.serviceAccount.name }} + automountServiceAccountToken: false + containers: + - name: app + {{- $imageTag := .Values.image.tag }} + {{- if eq .Values.environment "prod" }} + {{- $imageTag = regexReplaceAll "-rc\\.[0-9]+$" .Values.image.tag "" }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ $imageTag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/bin/sleep", "31536000"] + ports: + - name: http + containerPort: 8080 + resources: + requests: + cpu: {{ .Values.resources.requests.cpu | quote }} + memory: {{ .Values.resources.requests.memory | quote }} + limits: + memory: {{ .Values.resources.limits.memory | quote }} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/service.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/service.yaml new file mode 100644 index 000000000..520280ed0 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.nameOverride }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +spec: + selector: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/serviceaccount.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/serviceaccount.yaml new file mode 100644 index 000000000..2cf64e696 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.serviceAccount.name }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} +{{- with .Values.serviceAccount.imagePullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ .name }} +{{- end }} +{{- end }} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/values.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/values.yaml new file mode 100644 index 000000000..99423acb6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/charts/sample-component/values.yaml @@ -0,0 +1,30 @@ +replicaCount: 1 +environment: "" +nameOverride: "" + +application: + name: sample-app + +component: + name: sample-component + +image: + repository: ghcr.io/oracle/oraclelinux + tag: "9" + pullPolicy: IfNotPresent + +serviceAccount: + name: "" + automountToken: false + imagePullSecrets: + - name: ocirsecret + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 128Mi + +service: + port: 80 diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/values.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/values.yaml new file mode 100644 index 000000000..0090ae1c2 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-chart/sample-app/values.yaml @@ -0,0 +1 @@ +# Shared namespace baseline values belong here. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/application.env b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/application.env new file mode 100644 index 000000000..f24f1e4c5 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/application.env @@ -0,0 +1 @@ +component_name=sample-component diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/pull-request-pipeline.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/pull-request-pipeline.yaml new file mode 100644 index 000000000..aff784801 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/.oci-devops/pull-request-pipeline.yaml @@ -0,0 +1,12 @@ +version: 0.1 +component: build +timeoutInSeconds: 28799 +shell: bash + +steps: + - type: Command + name: Run component tests + timeoutInSeconds: 28799 + command: | + set -euo pipefail + echo "Unit tests succeeded" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/Dockerfile b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/Dockerfile new file mode 100644 index 000000000..4e8fc5677 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/Dockerfile @@ -0,0 +1,6 @@ +FROM container-registry.oracle.com/os/oraclelinux:10-slim + +LABEL org.opencontainers.image.title="sample-component" +LABEL org.opencontainers.image.description="Sample application image for the OKE Helm starter stack" + +CMD ["/usr/bin/sleep", "31536000"] diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/README.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/README.md new file mode 100644 index 000000000..0095be621 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/application-source/README.md @@ -0,0 +1,39 @@ +# Component Source Repository + +This repository is the generated source repository for one application component. + +For the default sample it is named `sample-component`. It contains starter application code, a Dockerfile, and the component-owned pull request pipeline build spec. + +## Important Files + +- `Dockerfile`: image build definition used by `-build`. +- `.oci-devops/application.env`: component identity read by the shared build pipeline. +- `.oci-devops/pull-request-pipeline.yaml`: component-owned PR validation build spec. + +Do not put Helm charts or values here. Chart changes belong in the application chart repository, by default `-chart`. + +## Developer Responsibilities + +The pull request pipeline spec is intentionally a placeholder. Testing strategies depend on the component language, framework, and integration boundaries, so replace `.oci-devops/pull-request-pipeline.yaml` with the checks that make sense for this component. + +The shared build pipeline expects the component build to be described by the `Dockerfile`. Prefer a multi-stage Dockerfile so compilation, dependency installation, and build tooling stay in builder stages while the final runtime image stays small and production-focused. + +## Development Flow + +1. Open a pull request to `main`. +2. `-pr` runs the component-owned checks. +3. After merge, `-build` builds a multi-architecture image tagged with the 7-character commit SHA. +4. The successful build deploys the dev release. +5. Run `-release-build` with a release candidate tag such as `1.0.0-rc.1` to promote the selected image. + +The build pipeline reads this contract file: + +```bash +component_name= +``` + +Images are pushed to: + +```text +.ocir.io////: +``` diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/deploy-cluster.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/deploy-cluster.py new file mode 100644 index 000000000..1d6aef2d9 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/deploy-cluster.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +def run(command, **kwargs): + print("+", " ".join(str(item) for item in command), flush=True) + return subprocess.run(command, check=True, universal_newlines=True, **kwargs) + + +def dependency_waves(targets): + pending = {target["tool"]: target for target in targets} + completed = set() + while pending: + ready = [ + target for target in pending.values() + if all(dependency not in pending or dependency in completed for dependency in target["depends_on"]) + ] + if not ready: + raise RuntimeError(f"Unable to order selected tools: {', '.join(sorted(pending))}") + yield sorted(ready, key=lambda target: target["tool"]) + for target in ready: + pending.pop(target["tool"]) + completed.add(target["tool"]) + + +def deploy_tool(args, target, work_dir): + tool = target["tool"] + namespace = target["namespace"] + actions = set(target["actions"]) + tool_dir = args.repo / "clusters" / args.cluster / "tools" / tool + + if "helm" in actions: + values_file = work_dir / f"{tool}-values.yaml" + run([ + "oci", "artifacts", "generic", "artifact", "download-by-path", + "--repository-id", args.artifact_repository_id, + "--artifact-path", f"cluster-admin/{args.cluster}/tools/{tool}/values.yaml", + "--artifact-version", args.commit, + "--file", str(values_file), + ]) + chart_name = target["chart"].rstrip("/").rsplit("/", 1)[-1] + chart_url = ( + f"oci://{args.registry}/{args.tenancy_namespace}/" + f"{args.chart_prefix}/{chart_name}" + ) + run([ + "helm", "upgrade", "--install", tool, chart_url, + "--version", target["chart_version"], + "--namespace", namespace, + "--create-namespace", + "--values", str(values_file), + "--atomic", + "--wait", + "--timeout", "10m", + "--history-max", "10", + ]) + + if "resources" in actions: + resources_dir = tool_dir / "resources" + manifests = sorted(resources_dir.glob("*.y*ml")) if resources_dir.is_dir() else [] + if manifests: + run([ + "kubectl", "apply", "--server-side", + "--field-manager=oci-devops-cluster-admin", + "--namespace", namespace, + "--filename", str(resources_dir), + ]) + else: + print(f"No supplemental resources to apply for {args.cluster}/{tool}") + + verify = tool_dir / "verify.sh" + if verify.is_file(): + environment = dict(os.environ, NAMESPACE=namespace) + run(["bash", str(verify)], env=environment) + + run(["helm", "status", tool, "--namespace", namespace]) + + +def apply_baseline(args): + baseline_dir = args.repo / "clusters" / args.cluster / "baseline" + manifests = sorted(baseline_dir.glob("*.y*ml")) + if not manifests: + print(f"No baseline manifests to apply for {args.cluster}") + return + run([ + "kubectl", "apply", "--server-side", + "--field-manager=oci-devops-cluster-admin", + "--filename", str(baseline_dir), + ]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--targets", type=Path, required=True) + parser.add_argument("--cluster", default="prod") + parser.add_argument("--commit", required=True) + parser.add_argument("--artifact-repository-id", required=True) + parser.add_argument("--registry", required=True) + parser.add_argument("--tenancy-namespace", required=True) + parser.add_argument("--chart-prefix", required=True) + args = parser.parse_args() + + selection = json.loads(args.targets.read_text(encoding="utf-8")) + if selection["commit"] != args.commit: + raise RuntimeError("Validated target commit does not match the deployment commit") + targets = [target for target in selection["targets"] if target["cluster"] == args.cluster] + tool_targets = [target for target in targets if target["kind"] == "tool"] + with tempfile.TemporaryDirectory() as temp_dir: + work_dir = Path(temp_dir) + for number, wave in enumerate(dependency_waves(tool_targets), start=1): + print( + f"Deploying {args.cluster} tool wave {number}: " + f"{', '.join(target['tool'] for target in wave)}", + flush=True, + ) + with ThreadPoolExecutor(max_workers=len(wave)) as executor: + futures = [ + executor.submit(deploy_tool, args, target, work_dir) + for target in wave + ] + for future in futures: + future.result() + + if any(target["kind"] == "baseline" for target in targets): + apply_baseline(args) + print(f"Cluster administration completed for {args.cluster} at {args.commit}") + + +if __name__ == "__main__": + try: + main() + except (KeyError, OSError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"Cluster deployment failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/mirror-charts.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/mirror-charts.py new file mode 100644 index 000000000..37cd90a82 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/mirror-charts.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + + +def operation_tags(**dimensions): + tags = { + "owner": "cluster-administrators", + "purpose": "cluster-administration", + "scope": "operations", + } + tags.update({key: value for key, value in dimensions.items() if value}) + return tags + + +def run(command, **kwargs): + print("+", " ".join(command)) + return subprocess.run(command, check=True, text=True, **kwargs) + + +def output(command): + return subprocess.check_output(command, text=True).strip() + + +def source_reference(repository, chart, alias): + if repository.startswith("oci://"): + return f"{repository.rstrip('/')}/{chart.lstrip('/')}" + return f"{alias}/{chart}" + + +def target_chart_name(chart): + return chart.rstrip("/").rsplit("/", 1)[-1] + + +def only_path(paths, description): + paths = list(paths) + if len(paths) != 1: + raise ValueError(f"Expected exactly one {description}, found {len(paths)}") + return paths[0] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--catalog", required=True) + parser.add_argument("--compartment-id", required=True) + parser.add_argument("--registry", required=True) + parser.add_argument("--tenancy-namespace", required=True) + parser.add_argument("--target-prefix", required=True) + args = parser.parse_args() + + catalog = yaml.safe_load(Path(args.catalog).read_text(encoding="utf-8")) + tools = catalog["tools"] + token_response = output( + ["oci", "raw-request", "--http-method", "GET", "--target-uri", f"https://{args.registry}/20180419/docker/token"] + ) + payload = json.loads(token_response) + if isinstance(payload.get("data"), str): + payload = json.loads(payload["data"]) + elif isinstance(payload.get("data"), dict): + payload = payload["data"] + token = payload["token"] + run(["helm", "registry", "login", args.registry, "-u", "BEARER_TOKEN", "--password-stdin"], input=token) + + target_root = f"oci://{args.registry}/{args.tenancy_namespace}/{args.target_prefix}" + with tempfile.TemporaryDirectory() as temp_dir: + for tool_name, chart in sorted(tools.items()): + version = chart["version"] + target_name = target_chart_name(chart["chart"]) + target = f"{target_root}/{target_name}" + exists = subprocess.run( + ["helm", "show", "chart", target, "--version", version], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode == 0 + if exists: + print(f"Chart already mirrored: {tool_name}:{version}") + continue + + repository_path = f"{args.target_prefix}/{target_name}" + existing_id = output([ + "oci", "artifacts", "container", "repository", "list", + "--compartment-id", args.compartment_id, + "--display-name", repository_path, + "--limit", "1", + "--query", "data.items[0].id", + "--raw-output", + ]) + if existing_id in ("", "null", "None"): + run([ + "oci", "artifacts", "container", "repository", "create", + "--compartment-id", args.compartment_id, + "--display-name", repository_path, + "--freeform-tags", json.dumps(operation_tags(role="chart-repository", tool=tool_name)), + ], stdout=subprocess.DEVNULL) + + repository = chart["repository"] + alias = f"mirror-{tool_name}" + if repository.startswith("https://"): + run(["helm", "repo", "add", alias, repository, "--force-update"]) + + source = source_reference(repository, chart["chart"], alias) + extract_dir = Path(temp_dir) / tool_name + extract_dir.mkdir() + run([ + "helm", "pull", source, + "--version", version, + "--untar", + "--untardir", str(extract_dir), + ]) + + chart_dir = only_path( + (path for path in extract_dir.iterdir() if path.is_dir()), + f"extracted chart directory for {tool_name}", + ) + chart_yaml = chart_dir / "Chart.yaml" + chart_metadata = yaml.safe_load(chart_yaml.read_text(encoding="utf-8")) + annotations = chart_metadata.get("annotations", {}) + safe_annotations = { + key: value for key, value in annotations.items() if str(value).isascii() + } + if safe_annotations != annotations: + print(f"Removed non-ASCII OCI annotations before mirroring {tool_name}:{version}") + if safe_annotations: + chart_metadata["annotations"] = safe_annotations + else: + chart_metadata.pop("annotations", None) + chart_yaml.write_text( + yaml.safe_dump(chart_metadata, sort_keys=False), encoding="utf-8" + ) + + package_dir = Path(temp_dir) / "packages" + package_dir.mkdir(exist_ok=True) + existing_archives = set(package_dir.glob("*.tgz")) + run(["helm", "package", str(chart_dir), "--destination", str(package_dir)]) + archive = only_path( + set(package_dir.glob("*.tgz")) - existing_archives, + f"packaged chart archive for {tool_name}", + ) + run(["helm", "push", str(archive), target_root]) + run(["helm", "show", "chart", target, "--version", version], stdout=subprocess.DEVNULL) + print(f"Mirrored chart: {tool_name}:{version}") + + +if __name__ == "__main__": + try: + main() + except (KeyError, OSError, ValueError, subprocess.CalledProcessError, yaml.YAMLError) as exc: + print(f"Chart mirroring failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/publish-and-dispatch.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/publish-and-dispatch.py new file mode 100644 index 000000000..e73d72560 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/publish-and-dispatch.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def operation_tags(**dimensions): + tags = { + "owner": "cluster-administrators", + "purpose": "cluster-administration", + "scope": "operations", + } + tags.update({key: value for key, value in dimensions.items() if value}) + return tags + + +def oci(arguments): + command = ["oci", *arguments, "--output", "json"] + print("+", " ".join(command), flush=True) + return json.loads(subprocess.check_output(command, text=True)) + + +def ensure_mirrors(pipeline_id): + result = oci([ + "devops", "build-run", "create", + "--build-pipeline-id", pipeline_id, + "--display-name", "Mirror cluster tool charts", + "--freeform-tags", json.dumps(operation_tags(role="chart-mirror")), + "--wait-for-state", "SUCCEEDED", + "--wait-for-state", "FAILED", + "--max-wait-seconds", "3600", + ]) + state = result["data"]["lifecycle-state"] + if state != "SUCCEEDED": + raise RuntimeError(f"Chart mirror build finished in state {state}") + + +def upload_artifact(repository_id, compartment_id, artifact_path, version, source, label): + existing = oci([ + "artifacts", "generic", "artifact", "list", + "--compartment-id", compartment_id, + "--repository-id", repository_id, + "--artifact-path", artifact_path, + "--artifact-version", version, + "--limit", "1", + ])["data"]["items"] + if existing: + print(f"{label} artifact already exists: {artifact_path}:{version}") + return + oci([ + "artifacts", "generic", "artifact", "upload-by-path", + "--repository-id", repository_id, + "--artifact-path", artifact_path, + "--artifact-version", version, + "--content-body", str(source), + ]) + print(f"Published {label} artifact: {artifact_path}:{version}") + + +def resource_tags(resource): + return resource.get("freeform-tags") or resource.get("free-form-tags") or {} + + +def pipeline_ids_by_cluster(project_id): + pipelines = oci([ + "devops", "deploy-pipeline", "list", + "--project-id", project_id, + "--all", + ])["data"]["items"] + result = {} + for pipeline in pipelines: + tags = resource_tags(pipeline) + cluster = tags.get("cluster") + if ( + tags.get("purpose") == "cluster-administration" + and tags.get("role") == "cluster-pipeline" + and cluster + ): + result[cluster] = pipeline["id"] + elif pipeline.get("display-name", "").startswith("cluster-admin-"): + result.setdefault(pipeline["display-name"].removeprefix("cluster-admin-"), pipeline["id"]) + return result + + +def pipeline_arguments(pipeline_id, commit): + pipeline = oci([ + "devops", "deploy-pipeline", "get", + "--pipeline-id", pipeline_id, + ])["data"] + parameters = pipeline.get("deploy-pipeline-parameters", {}).get("items", []) + arguments = { + parameter["name"]: parameter.get("default-value") or "unused" + for parameter in parameters + } + arguments["config_commit"] = commit + return arguments + + +def run_pipeline(pipeline_id, cluster, commit, arguments, plan_summary=""): + display_name = f"Deploy {cluster} changes {commit[:7]}" + if plan_summary: + display_name = f"{display_name}: {plan_summary}"[:255] + deployment_arguments = { + "items": [{"name": name, "value": value} for name, value in sorted(arguments.items())] + } + result = oci([ + "devops", "deployment", "create-pipeline-deployment", + "--pipeline-id", pipeline_id, + "--display-name", display_name, + "--deployment-arguments", json.dumps(deployment_arguments), + "--freeform-tags", json.dumps( + operation_tags( + cluster=cluster, + role="cluster-deployment", + config_commit=commit, + targets=plan_summary[:255], + ) + ), + "--wait-for-state", "SUCCEEDED", + "--wait-for-state", "FAILED", + "--max-wait-seconds", "36000", + ]) + state = result["data"]["lifecycle-state"] + if state != "SUCCEEDED": + raise RuntimeError(f"Cluster deployment for {cluster} finished in state {state}") + print(f"Completed cluster deployment: {cluster}") + + +def dependency_waves(targets): + pending = {target["tool"]: target for target in targets} + completed = set() + while pending: + ready = [ + target for target in pending.values() + if all(dependency not in pending or dependency in completed for dependency in target["depends_on"]) + ] + if not ready: + raise RuntimeError(f"Unable to order selected tools: {', '.join(sorted(pending))}") + yield sorted(ready, key=lambda target: target["tool"]) + for target in ready: + pending.pop(target["tool"]) + completed.add(target["tool"]) + + +def cluster_plan(cluster, targets, commit): + lines = [f"Cluster: {cluster}", f"Configuration commit: {commit}"] + tool_targets = [target for target in targets if target["kind"] == "tool"] + for number, wave in enumerate(dependency_waves(tool_targets), start=1): + lines.append(f"Wave {number}:") + for target in wave: + actions = " + ".join(target["actions"]) + details = [f"namespace={target['namespace']}"] + if "helm" in target["actions"]: + details.extend(( + f"chart={target['chart_version']}", + f"values={commit}", + )) + lines.append(f" - {target['tool']}: {actions} ({', '.join(details)})") + if any(target["kind"] == "baseline" for target in targets): + lines.extend(("Final:", " - cluster baseline")) + if len(lines) == 2: + lines.append("No deployment stages selected") + return "\n".join(lines) + + +def plan_summary(targets): + parts = [] + for target in sorted(targets, key=lambda item: (item["kind"], item.get("tool", ""))): + if target["kind"] == "baseline": + parts.append("baseline") + else: + parts.append(f"{target['tool']}[{'+'.join(target['actions'])}]") + return ", ".join(parts) + + +def print_execution_plan(selection): + targets_by_cluster = {} + for target in selection["targets"]: + targets_by_cluster.setdefault(target["cluster"], []).append(target) + print("\n=== Cluster Administration Execution Plan ===", flush=True) + if not targets_by_cluster: + print(f"Configuration commit: {selection['commit']}") + print("No cluster deployment stages selected", flush=True) + return + for cluster, targets in sorted(targets_by_cluster.items()): + print(cluster_plan(cluster, targets, selection["commit"]), flush=True) + print(flush=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--targets", required=True) + parser.add_argument("--project-id", required=True) + parser.add_argument("--mirror-pipeline-id", required=True) + parser.add_argument("--artifact-repository-id", required=True) + parser.add_argument("--compartment-id", required=True) + args = parser.parse_args() + + repo = Path(args.repo).resolve() + selection = json.loads(Path(args.targets).read_text(encoding="utf-8")) + commit = selection["commit"] + print_execution_plan(selection) + targets_by_cluster = {} + for target in selection["targets"]: + targets_by_cluster.setdefault(target["cluster"], []).append(target) + + helm_targets = [ + target for target in selection["targets"] + if target["kind"] == "tool" and "helm" in target["actions"] + ] + if selection["catalog_changed"] or helm_targets: + ensure_mirrors(args.mirror_pipeline_id) + + for target in helm_targets: + cluster = target["cluster"] + tool = target["tool"] + upload_artifact( + args.artifact_repository_id, + args.compartment_id, + f"cluster-admin/{cluster}/tools/{tool}/values.yaml", + commit, + repo / "clusters" / cluster / "tools" / tool / "values.yaml", + "values", + ) + + if targets_by_cluster: + upload_artifact( + args.artifact_repository_id, + args.compartment_id, + "cluster-admin/deployment-plan.json", + commit, + Path(args.targets), + "deployment plan", + ) + + pipeline_ids = pipeline_ids_by_cluster(args.project_id) + for cluster, targets in sorted(targets_by_cluster.items()): + pipeline_id = pipeline_ids.get(cluster) + if not pipeline_id: + raise RuntimeError(f"Cluster deployment pipeline not found: cluster-admin-{cluster}") + base_arguments = pipeline_arguments(pipeline_id, commit) + + run_pipeline( + pipeline_id, + cluster, + commit, + base_arguments, + plan_summary(targets), + ) + + +if __name__ == "__main__": + try: + main() + except (KeyError, OSError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"Publish and dispatch failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/validate-config.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/validate-config.py new file mode 100644 index 000000000..b8707eb58 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/cluster-admin/script/validate-config.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +import argparse +from collections import defaultdict +import json +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +DNS_LABEL = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]*$") + + +def fail(message): + raise ValueError(message) + + +def load_yaml(path): + with path.open(encoding="utf-8") as stream: + return yaml.safe_load(stream) + + +def validate_label(value, field): + if not isinstance(value, str) or len(value) > 63 or not DNS_LABEL.fullmatch(value): + fail(f"{field} must be a Kubernetes DNS label of at most 63 characters: {value!r}") + + +def changed_files(repo, commit, mode): + if mode == "all": + return [str(path.relative_to(repo)) for path in repo.rglob("*") if path.is_file()] + try: + parent_count = subprocess.check_output( + ["git", "-C", str(repo), "rev-list", "--parents", "-n", "1", commit], text=True + ).strip().split() + if len(parent_count) < 2: + return [str(path.relative_to(repo)) for path in repo.rglob("*") if path.is_file()] + output = subprocess.check_output( + ["git", "-C", str(repo), "diff", "--name-only", f"{commit}^", commit], text=True + ) + return [line for line in output.splitlines() if line] + except subprocess.CalledProcessError as exc: + fail(f"Unable to determine files changed by commit {commit}: {exc}") + + +def validate(repo, commit, mode): + clusters = {"noprod", "prod"} + + catalog = load_yaml(repo / "catalog/tools.yaml") + tools = catalog.get("tools") if isinstance(catalog, dict) else None + if not isinstance(tools, dict): + fail("catalog/tools.yaml must contain a tools mapping") + + for tool_name, chart in tools.items(): + validate_label(tool_name, f"catalog tool {tool_name}") + if not isinstance(chart, dict): + fail(f"Catalog entry {tool_name} must be an object") + if not isinstance(chart.get("chart"), str) or not chart["chart"].strip(): + fail(f"Catalog tool {tool_name} must define an upstream chart name") + if not isinstance(chart.get("repository"), str) or not chart["repository"].startswith(("https://", "oci://")): + fail(f"Catalog tool {tool_name} must use an HTTPS Helm repository or OCI chart repository") + if not isinstance(chart.get("version"), str) or not VERSION.fullmatch(chart["version"]): + fail(f"Catalog tool {tool_name} has an invalid chart version") + + configured = {} + for cluster_name in sorted(clusters): + tools_dir = repo / "clusters" / cluster_name / "tools" + for tool_dir in sorted(path for path in tools_dir.glob("*") if path.is_dir()): + metadata_path = tool_dir / "tool.yaml" + metadata = load_yaml(metadata_path) + if not isinstance(metadata, dict): + fail(f"Missing or invalid tool metadata: {metadata_path.relative_to(repo)}") + tool_name = metadata.get("name") + namespace = metadata.get("namespace", tool_name) + dependencies = metadata.get("depends_on", []) + if tool_name != tool_dir.name: + fail(f"{metadata_path.relative_to(repo)} name must match its directory: {tool_dir.name}") + validate_label(tool_name, f"{cluster_name} tool name") + validate_label(namespace, f"{cluster_name}/{tool_name} namespace") + if not isinstance(dependencies, list) or not all(isinstance(item, str) for item in dependencies): + fail(f"{metadata_path.relative_to(repo)} depends_on must be a list of tool names") + if len(dependencies) != len(set(dependencies)): + fail(f"{metadata_path.relative_to(repo)} contains duplicate dependencies") + for dependency in dependencies: + validate_label(dependency, f"{cluster_name}/{tool_name} dependency") + if dependency == tool_name: + fail(f"Tool {cluster_name}/{tool_name} cannot depend on itself") + if tool_name not in tools: + fail(f"Tool {tool_name} configured for {cluster_name} is missing from catalog/tools.yaml") + values_path = tool_dir / "values.yaml" + values = load_yaml(values_path) + if values is not None and not isinstance(values, dict): + fail(f"{values_path.relative_to(repo)} must contain a YAML mapping") + resources_dir = values_path.parent / "resources" + if resources_dir.exists(): + for manifest in sorted(resources_dir.glob("*.y*ml")): + with manifest.open(encoding="utf-8") as stream: + for document in yaml.safe_load_all(stream): + if document is None: + continue + if not isinstance(document, dict) or not document.get("apiVersion") or not document.get("kind"): + fail(f"Invalid Kubernetes document in {manifest.relative_to(repo)}") + if document["kind"] == "Secret": + fail(f"Plain Secret objects are not allowed: {manifest.relative_to(repo)}; use ExternalSecret") + declared_namespace = document.get("metadata", {}).get("namespace") + if declared_namespace and declared_namespace != namespace: + fail( + f"{manifest.relative_to(repo)} targets namespace {declared_namespace}, expected {namespace}" + ) + configured[(cluster_name, tool_name)] = { + "chart": tools[tool_name]["chart"], + "cluster": cluster_name, + "kind": "tool", + "tool": tool_name, + "namespace": namespace, + "chart_version": tools[tool_name]["version"], + "depends_on": dependencies, + } + baseline_dir = repo / "clusters" / cluster_name / "baseline" + if not baseline_dir.is_dir(): + fail(f"Missing baseline directory: {baseline_dir.relative_to(repo)}") + for manifest in sorted(baseline_dir.glob("*.y*ml")): + with manifest.open(encoding="utf-8") as stream: + for document in yaml.safe_load_all(stream): + if document is None: + continue + if not isinstance(document, dict) or not document.get("apiVersion") or not document.get("kind"): + fail(f"Invalid Kubernetes document in {manifest.relative_to(repo)}") + if document.get("metadata", {}).get("namespace"): + fail(f"Baseline object must not declare metadata.namespace: {manifest.relative_to(repo)}") + + for (cluster_name, tool_name), tool in sorted(configured.items()): + for dependency in tool["depends_on"]: + if (cluster_name, dependency) not in configured: + fail(f"Tool {cluster_name}/{tool_name} depends on unknown tool {dependency}") + + visiting = set() + visited = set() + + def visit(key, path): + if key in visiting: + cycle = " -> ".join([*(item[1] for item in path), key[1]]) + fail(f"Tool dependency cycle in {key[0]}: {cycle}") + if key in visited: + return + visiting.add(key) + for dependency in configured[key]["depends_on"]: + visit((key[0], dependency), [*path, key]) + visiting.remove(key) + visited.add(key) + + for key in sorted(configured): + visit(key, []) + + files = changed_files(repo, commit, mode) + baseline_targets = set() + target_actions = defaultdict(set) + catalog_changed = any(path == "catalog/tools.yaml" for path in files) + for path in files: + parts = Path(path).parts + if len(parts) < 3 or parts[0] != "clusters": + continue + cluster_name = parts[1] + if cluster_name not in clusters: + fail(f"Unknown cluster path in changed file: {path}") + if parts[2] == "baseline": + if Path(path).name != "README.md": + baseline_targets.add(cluster_name) + elif len(parts) >= 5 and parts[2] == "tools": + tool_name = parts[3] + key = (cluster_name, tool_name) + if key not in configured: + fail(f"Changed path has no valid tool.yaml metadata: {path}") + relative_parts = parts[4:] + if Path(path).name == "README.md": + continue + if relative_parts[0] in ("tool.yaml", "values.yaml"): + target_actions[key].add("helm") + elif relative_parts[0] == "resources" or relative_parts[0] == "verify.sh": + target_actions[key].add("resources") + else: + fail(f"Unsupported tool configuration path: {path}") + else: + fail(f"Unsupported cluster configuration path: {path}") + + if catalog_changed: + for key in configured: + target_actions[key].add("helm") + + # A changed prerequisite selects all downstream tools so dependency ordering + # remains meaningful during incremental orchestrated deployments. + selected = set(target_actions) + expanded = True + while expanded: + expanded = False + for key, tool in configured.items(): + dependency_keys = {(key[0], dependency) for dependency in tool["depends_on"]} + if key not in selected and dependency_keys.intersection(selected): + selected.add(key) + target_actions[key].update(("helm", "resources")) + expanded = True + + targets = [{"cluster": cluster, "kind": "baseline"} for cluster in sorted(baseline_targets)] + action_order = {"helm": 0, "resources": 1} + for key in sorted(selected): + target = dict(configured[key]) + target["actions"] = sorted(target_actions[key], key=action_order.get) + targets.append(target) + + return { + "catalog_changed": catalog_changed, + "commit": commit, + "targets": sorted(targets, key=lambda item: (item["cluster"], item["kind"], item.get("tool", ""))), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--commit", required=True) + parser.add_argument("--mode", choices=("changed", "all"), default="changed") + parser.add_argument("--output", required=True) + args = parser.parse_args() + result = validate(Path(args.repo).resolve(), args.commit, args.mode) + Path(args.output).write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(f"Validated cluster administration configuration; selected {len(result['targets'])} target(s)") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, json.JSONDecodeError, yaml.YAMLError) as exc: + print(f"Configuration validation failed: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/README.md b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/README.md new file mode 100644 index 000000000..1831086d8 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/README.md @@ -0,0 +1,28 @@ +# Pipelines + +This repository contains reusable OCI DevOps build specs and helper scripts generated by the OKE DevOps Starter stack. + +It is the shared source for pipeline logic. Component source repositories and application chart repositories are checked out as secondary sources only when a pipeline needs them. + +## Generated Build Specs + +For each component: + +- `-build-pipeline.yaml` builds source changes, packages the component chart when needed, and exports parameters for dev deployment. +- `-release-pipeline.yaml` creates the OCI DevOps release candidate Git tag, retags the matching 7-character SHA image as a SemVer RC such as `1.0.0-rc.1`, verifies image digests, and starts the component release deployment. + +For each application: + +- `-package-pipeline.yaml` packages the application umbrella baseline chart and starts the application baseline deployment pipeline. + +Shared: + +- `helm-chart-pipeline.yaml` is a generic chart packager retained for future reuse. + +Resource Manager treats these files as starter content. New application/component specifications are added when their paths are missing, while existing files and shared scripts are never overwritten by later stack applies. + +## Style + +Build specs should stay readable and macro-oriented. Detailed branching, registry checks, chart parsing, image promotion, and release input resolution belong in `script/*.sh`. + +Default sample pipeline names are `sample-component-build`, `sample-app-package`, `sample-component-pr`, and `sample-component-release-build`. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/helm-chart-pipeline.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/helm-chart-pipeline.yaml new file mode 100644 index 000000000..c477a8fd2 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/helm-chart-pipeline.yaml @@ -0,0 +1,40 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + CHART_VALUES_FILE: "" + repo_compartment_id: "" + region: "" + region_key: "" + tenancy_namespace: "" + +steps: + - type: Command + name: Package and push Helm chart + failImmediatelyOnError: true + command: | + export OCI_CLI_REGION="${region}" + + : "${chart_name:?chart_name pipeline parameter is required}" + : "${chart_version:?chart_version pipeline parameter is required}" + : "${chart_path:?chart_path pipeline parameter is required}" + : "${chart_source_dir:?chart_source_dir pipeline parameter is required}" + : "${chart_repo_prefix:?chart_repo_prefix pipeline parameter is required}" + cd "${OCI_WORKSPACE_DIR}/${chart_source_dir}" + args=( + -c "${repo_compartment_id}" + -k "${region_key}" + -n "${chart_name}" + -v "${chart_version}" + -d "${chart_path}" + -p "${chart_repo_prefix}" + -t "${tenancy_namespace}" + ) + if [ -n "${CHART_VALUES_FILE:-}" ]; then + args+=(-f "${CHART_VALUES_FILE}") + fi + "${OCI_PRIMARY_SOURCE_DIR}/script/package-push-chart.sh" \ + "${args[@]}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-build-image.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-build-image.sh new file mode 100644 index 000000000..1b0fd03a2 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-build-image.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +write_delivery_env() { + { + printf "BUILD_IMAGE=%s\n" "$BUILD_IMAGE" + printf "PACKAGE_CHART=%s\n" "$PACKAGE_CHART" + printf "CHECK_CHART=%s\n" "$CHECK_CHART" + printf "source_dir=%q\n" "$source_dir" + printf "chart_source_dir=%q\n" "$chart_source_dir" + printf "chart_path=%q\n" "$chart_path" + printf "chart_name=%q\n" "$chart_name" + printf "chart_repo_prefix=%q\n" "$chart_repo_prefix" + printf "application_env_file=%q\n" "$application_env_file" + printf "application_validated_env_file=%q\n" "$application_validated_env_file" + printf "chart_values_file=%q\n" "$chart_values_file" + printf "chart_version=%q\n" "$chart_version" + } > "$DELIVERY_ENV_FILE" +} + +unset -v DELIVERY_ENV_FILE +unset -v COMPARTMENT_ID +unset -v OCIR_REGION_KEY +unset -v IMAGE_REPOSITORY_PATH +unset -v TENANCY_NAMESPACE + +while getopts e:c:k:p:t: flag; do + case "${flag}" in + e) DELIVERY_ENV_FILE=${OPTARG} ;; + c) COMPARTMENT_ID=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + p) IMAGE_REPOSITORY_PATH=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${DELIVERY_ENV_FILE:-}" ] || [ -z "${COMPARTMENT_ID:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${IMAGE_REPOSITORY_PATH:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DELIVERY_ENV_FILE" + +if [ "$BUILD_IMAGE" != "true" ]; then + echo "Skipping image build." + exit 0 +fi + +cd "${OCI_WORKSPACE_DIR}/${source_dir}" +bash "${SCRIPT_DIR}/read-application-metadata.sh" \ + -f "$application_env_file" \ + -o "$application_validated_env_file" +source "$application_validated_env_file" + +bash "${SCRIPT_DIR}/build-push-image.sh" \ + -c "$COMPARTMENT_ID" \ + -k "$OCIR_REGION_KEY" \ + -n "$component_name" \ + -p "$IMAGE_REPOSITORY_PATH" \ + -t "${TENANCY_NAMESPACE:-}" + +write_delivery_env diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-check-chart.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-check-chart.sh new file mode 100644 index 000000000..db9d65f02 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-check-chart.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -euo pipefail + +write_delivery_env() { + { + printf "BUILD_IMAGE=%s\n" "$BUILD_IMAGE" + printf "PACKAGE_CHART=%s\n" "$PACKAGE_CHART" + printf "CHECK_CHART=%s\n" "$CHECK_CHART" + printf "source_dir=%q\n" "$source_dir" + printf "chart_source_dir=%q\n" "$chart_source_dir" + printf "chart_path=%q\n" "$chart_path" + printf "chart_name=%q\n" "$chart_name" + printf "chart_repo_prefix=%q\n" "$chart_repo_prefix" + printf "application_env_file=%q\n" "$application_env_file" + printf "application_validated_env_file=%q\n" "$application_validated_env_file" + printf "chart_values_file=%q\n" "$chart_values_file" + printf "chart_version=%q\n" "$chart_version" + } > "$DELIVERY_ENV_FILE" +} + +unset -v DELIVERY_ENV_FILE +unset -v OCIR_REGION_KEY +unset -v TENANCY_NAMESPACE + +while getopts e:k:t: flag; do + case "${flag}" in + e) DELIVERY_ENV_FILE=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${DELIVERY_ENV_FILE:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${TENANCY_NAMESPACE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +source "$DELIVERY_ENV_FILE" + +if [ "$CHECK_CHART" != "true" ]; then + echo "Skipping chart existence check." + exit 0 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ -z "${chart_version:-}" ]; then + chart_version="$(bash "${SCRIPT_DIR}/read-chart-version.sh" -p "${OCI_WORKSPACE_DIR}/${chart_source_dir}/${chart_path}")" + write_delivery_env +fi + +remote_registry="${OCIR_REGION_KEY}.ocir.io" +chart_url="oci://${remote_registry}/${TENANCY_NAMESPACE}/${chart_repo_prefix%/}/${chart_name}" + +# Helm authenticates to OCIR with the same bearer-token pattern used by Docker. +token="$(oci raw-request --http-method GET --target-uri "https://${remote_registry}/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" +if [ -z "$token" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 +fi + +echo "$token" | helm registry login "$remote_registry" -u BEARER_TOKEN --password-stdin +if helm show chart "$chart_url" --version "$chart_version" >/dev/null 2>&1; then + echo "Chart already exists: ${chart_url}:${chart_version}" +else + # First source builds need to publish the matching chart before releases can refer to it. + echo "Chart is missing; this run will package and upload it: ${chart_url}:${chart_version}" + PACKAGE_CHART=true + write_delivery_env +fi diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-package-chart.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-package-chart.sh new file mode 100644 index 000000000..cd0fc2037 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-package-chart.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -euo pipefail + +write_delivery_env() { + { + printf "BUILD_IMAGE=%s\n" "$BUILD_IMAGE" + printf "PACKAGE_CHART=%s\n" "$PACKAGE_CHART" + printf "CHECK_CHART=%s\n" "$CHECK_CHART" + printf "source_dir=%q\n" "$source_dir" + printf "chart_source_dir=%q\n" "$chart_source_dir" + printf "chart_path=%q\n" "$chart_path" + printf "chart_name=%q\n" "$chart_name" + printf "chart_repo_prefix=%q\n" "$chart_repo_prefix" + printf "application_env_file=%q\n" "$application_env_file" + printf "application_validated_env_file=%q\n" "$application_validated_env_file" + printf "chart_values_file=%q\n" "$chart_values_file" + printf "chart_version=%q\n" "$chart_version" + } > "$DELIVERY_ENV_FILE" +} + +unset -v DELIVERY_ENV_FILE +unset -v COMPARTMENT_ID +unset -v OCIR_REGION_KEY +unset -v TENANCY_NAMESPACE + +while getopts e:c:k:t: flag; do + case "${flag}" in + e) DELIVERY_ENV_FILE=${OPTARG} ;; + c) COMPARTMENT_ID=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${DELIVERY_ENV_FILE:-}" ] || [ -z "${COMPARTMENT_ID:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DELIVERY_ENV_FILE" + +if [ "$PACKAGE_CHART" != "true" ]; then + echo "Skipping chart packaging." + exit 0 +fi + +cd "${OCI_WORKSPACE_DIR}/${chart_source_dir}" +if [ -z "${chart_version:-}" ]; then + chart_version="$(bash "${SCRIPT_DIR}/read-chart-version.sh" -p "$chart_path")" + write_delivery_env +fi + +printf "Packaging chart version: %s\n" "$chart_version" +args=( + -c "$COMPARTMENT_ID" + -k "$OCIR_REGION_KEY" + -n "$chart_name" + -v "$chart_version" + -d "$chart_path" + -p "$chart_repo_prefix" + -t "${TENANCY_NAMESPACE:-}" +) +if [ -n "${chart_values_file:-}" ]; then + args+=(-f "$chart_values_file") +fi + +bash "${SCRIPT_DIR}/package-push-chart.sh" "${args[@]}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-actions.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-actions.sh new file mode 100644 index 000000000..6d8429e91 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-actions.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -euo pipefail + +write_delivery_env() { + { + printf "BUILD_IMAGE=%s\n" "$BUILD_IMAGE" + printf "PACKAGE_CHART=%s\n" "$PACKAGE_CHART" + printf "CHECK_CHART=%s\n" "$CHECK_CHART" + printf "source_dir=%q\n" "$SOURCE_DIR" + printf "chart_source_dir=%q\n" "$CHART_SOURCE_DIR" + printf "chart_path=%q\n" "$CHART_PATH" + printf "chart_name=%q\n" "$CHART_NAME" + printf "chart_repo_prefix=%q\n" "$CHART_REPO_PREFIX" + printf "application_env_file=%q\n" "$APPLICATION_ENV_FILE" + printf "application_validated_env_file=%q\n" "$APPLICATION_VALIDATED_ENV_FILE" + printf "chart_values_file=%q\n" "$CHART_VALUES_FILE" + printf "chart_version=%q\n" "" + } > "$OUTPUT_ENV_FILE" +} + +normalize_repo_url() { + # OCI DevOps exposes a clean trigger URL, but Git remotes include SERVICE credentials. + printf "%s" "$1" | sed -E 's#^(https://)[^/@]+(:[^/@]*)?@#\1#; s#\.git$##' +} + +find_application_sources() { + local candidate + for candidate in "${OCI_WORKSPACE_DIR}"/*; do + [ -d "$candidate" ] || continue + [ -f "${candidate}/.oci-devops/application.env" ] || continue + basename "$candidate" + done +} + +single_application_source() { + local sources=("$@") + if [ "${#sources[@]}" -ne 1 ]; then + echo "Unable to select one application source. Found: ${sources[*]:-none}" >&2 + exit 1 + fi + printf "%s\n" "${sources[0]}" +} + +unset -v SOURCE_DIR +unset -v CHART_SOURCE_DIR +unset -v CHART_PATH +unset -v CHART_REPO_PREFIX +unset -v OUTPUT_ENV_FILE + +while getopts s:a:p:r:o: flag; do + case "${flag}" in + s) SOURCE_DIR=${OPTARG} ;; + a) CHART_SOURCE_DIR=${OPTARG} ;; + p) CHART_PATH=${OPTARG} ;; + r) CHART_REPO_PREFIX=${OPTARG} ;; + o) OUTPUT_ENV_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${CHART_SOURCE_DIR:-}" ] || [ -z "${CHART_PATH:-}" ] || [ -z "${CHART_REPO_PREFIX:-}" ] || [ -z "${OUTPUT_ENV_FILE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" + +application_sources=() +while IFS= read -r source_name; do + application_sources+=("$source_name") +done < <(find_application_sources) +CHART_NAME="$(basename "$CHART_PATH")" +APPLICATION_ENV_FILE=".oci-devops/application.env" +APPLICATION_VALIDATED_ENV_FILE=".oci-devops/application.validated.env" +CHART_VALUES_FILE="" + +trigger_source_url="$(normalize_repo_url "${OCI_TRIGGER_SOURCE_URL:-}")" +chart_remote_url="$(normalize_repo_url "$(git -C "${OCI_WORKSPACE_DIR}/${CHART_SOURCE_DIR}" config --get remote.origin.url || true)")" + +BUILD_IMAGE=false +PACKAGE_CHART=false +CHECK_CHART=false + +if [ -z "${OCI_TRIGGER_SOURCE_URL:-}" ]; then + SOURCE_DIR="${SOURCE_DIR:-$(single_application_source "${application_sources[@]}")}" + BUILD_IMAGE=true + PACKAGE_CHART=true +elif [ "$trigger_source_url" = "$chart_remote_url" ]; then + SOURCE_DIR="${SOURCE_DIR:-$(single_application_source "${application_sources[@]}")}" + PACKAGE_CHART=true +else + for source_candidate in "${application_sources[@]}"; do + source_remote_url="$(normalize_repo_url "$(git -C "${OCI_WORKSPACE_DIR}/${source_candidate}" config --get remote.origin.url || true)")" + if [ "$trigger_source_url" = "$source_remote_url" ]; then + SOURCE_DIR="$source_candidate" + BUILD_IMAGE=true + CHECK_CHART=true + break + fi + done + + if [ -z "${SOURCE_DIR:-}" ]; then + echo "Unable to classify trigger source: ${OCI_TRIGGER_SOURCE_URL}" >&2 + echo "Application sources: ${application_sources[*]:-none}" >&2 + echo "Application chart URL: ${chart_remote_url}" >&2 + exit 1 + fi +fi + +write_delivery_env + +printf "Trigger source: %s\n" "${OCI_TRIGGER_SOURCE_URL:-manual}" +printf "Build image: %s\n" "$BUILD_IMAGE" +printf "Package chart: %s\n" "$PACKAGE_CHART" +printf "Check chart exists: %s\n" "$CHECK_CHART" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-component-chart.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-component-chart.sh new file mode 100644 index 000000000..a503e19e1 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/application-delivery-select-component-chart.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -euo pipefail + +write_component_env() { + { + printf "BUILD_IMAGE=%s\n" "$BUILD_IMAGE" + printf "PACKAGE_CHART=%s\n" "$PACKAGE_CHART" + printf "CHECK_CHART=%s\n" "$CHECK_CHART" + printf "source_dir=%q\n" "$source_dir" + printf "chart_source_dir=%q\n" "$chart_source_dir" + printf "chart_path=%q\n" "$COMPONENT_CHART_PATH" + printf "chart_name=%q\n" "$COMPONENT_NAME" + printf "chart_repo_prefix=%q\n" "$chart_repo_prefix" + printf "application_env_file=%q\n" "$application_env_file" + printf "application_validated_env_file=%q\n" "$application_validated_env_file" + printf "chart_values_file=%q\n" "" + printf "chart_version=%q\n" "" + } > "$OUTPUT_ENV_FILE" +} + +unset -v DELIVERY_ENV_FILE +unset -v COMPONENT_NAME +unset -v COMPONENT_CHART_PATH +unset -v OUTPUT_ENV_FILE + +while getopts e:n:p:o: flag; do + case "${flag}" in + e) DELIVERY_ENV_FILE=${OPTARG} ;; + n) COMPONENT_NAME=${OPTARG} ;; + p) COMPONENT_CHART_PATH=${OPTARG} ;; + o) OUTPUT_ENV_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${DELIVERY_ENV_FILE:-}" ] || [ -z "${COMPONENT_NAME:-}" ] || [ -z "${COMPONENT_CHART_PATH:-}" ] || [ -z "${OUTPUT_ENV_FILE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +source "$DELIVERY_ENV_FILE" + +write_component_env + +printf "Selected component chart: %s\n" "$COMPONENT_NAME" +printf "Component chart path: %s\n" "$COMPONENT_CHART_PATH" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/build-push-image.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/build-push-image.sh new file mode 100644 index 000000000..76444764d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/build-push-image.sh @@ -0,0 +1,151 @@ +#!/bin/bash +set -euo pipefail + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" >&2 + exit 1 + } +} + +validate_ocid() { + local value="$1" + [[ "$value" =~ ^ocid1\.[a-z0-9_-]+\.[a-z0-9_-]*\.[a-z0-9-]*\.[a-z0-9]+$ ]] +} + +validate_repo_path() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$ ]] +} + +validate_name() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +validate_region_key() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+$ ]] +} + +validate_namespace() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +unset -v COMPARTMENT_ID +unset -v OCIR_REGION_KEY +unset -v IMAGE_NAME +unset -v OCIR_REPOSITORY_PATH +unset -v TENANCY_NAMESPACE + +while getopts c:k:n:p:t: flag; do + case "${flag}" in + c) COMPARTMENT_ID=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + n) IMAGE_NAME=${OPTARG} ;; + p) OCIR_REPOSITORY_PATH=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${COMPARTMENT_ID:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${IMAGE_NAME:-}" ] || [ -z "${OCIR_REPOSITORY_PATH:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +require_cmd docker +require_cmd git +require_cmd oci + +if ! validate_ocid "$COMPARTMENT_ID"; then + echo "Invalid compartment OCID" >&2 + exit 1 +fi + +if ! validate_region_key "$OCIR_REGION_KEY"; then + echo "Invalid OCIR region key: ${OCIR_REGION_KEY}" >&2 + exit 1 +fi + +if ! validate_name "$IMAGE_NAME"; then + echo "Invalid image name: ${IMAGE_NAME}" >&2 + exit 1 +fi + +if ! validate_repo_path "$OCIR_REPOSITORY_PATH"; then + echo "Invalid OCIR repository path: ${OCIR_REPOSITORY_PATH}" >&2 + exit 1 +fi + +if [ -n "${TENANCY_NAMESPACE:-}" ] && ! validate_namespace "$TENANCY_NAMESPACE"; then + echo "Invalid tenancy namespace: ${TENANCY_NAMESPACE}" >&2 + exit 1 +fi + +if [ ! -f Dockerfile ]; then + echo "Dockerfile not found in $(pwd)" >&2 + exit 1 +fi + +IMAGE_TAG="$(git rev-parse --short=7 HEAD)" +IMAGE_REPOSITORY="${OCIR_REPOSITORY_PATH%/}" + +repo_namespace="${TENANCY_NAMESPACE:-}" +if [ -z "$repo_namespace" ]; then + repo_namespace=$(oci artifacts container configuration get --compartment-id "$COMPARTMENT_ID" --query "data.namespace" --raw-output) +fi + +if ! repo_id=$(oci artifacts container repository list --compartment-id "$COMPARTMENT_ID" --display-name "$IMAGE_REPOSITORY" --limit 1 --query "data.items[0].id" --raw-output 2>/dev/null); then + repo_id="" +fi + +if [ "$repo_id" = "null" ]; then + repo_id="" +fi + +if [ -z "$repo_id" ]; then + oci artifacts container repository create --display-name "$IMAGE_REPOSITORY" --compartment-id "$COMPARTMENT_ID" >/dev/null +fi + +REMOTE_REGISTRY="${OCIR_REGION_KEY}.ocir.io" +IMAGE_URI="${REMOTE_REGISTRY}/${repo_namespace}/${IMAGE_REPOSITORY}:${IMAGE_TAG}" + +TOKEN="$(oci raw-request --http-method GET --target-uri "https://${REMOTE_REGISTRY}/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" +if [ -z "$TOKEN" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 +fi + +echo "$TOKEN" | docker login "$REMOTE_REGISTRY" -u BEARER_TOKEN --password-stdin +# Publish a multi-architecture manifest without relying on Buildx. OCI DevOps +# currently ships an older Buildx plugin, while Docker's manifest command is +# enough for this starter image lifecycle. +AMD64_IMAGE_URI="${REMOTE_REGISTRY}/${repo_namespace}/${IMAGE_REPOSITORY}:${IMAGE_TAG}-amd64" +ARM64_IMAGE_URI="${REMOTE_REGISTRY}/${repo_namespace}/${IMAGE_REPOSITORY}:${IMAGE_TAG}-arm64" + +docker build --pull --platform linux/amd64 --tag "$AMD64_IMAGE_URI" . +docker push "$AMD64_IMAGE_URI" + +docker build --pull --platform linux/arm64 --tag "$ARM64_IMAGE_URI" . +docker push "$ARM64_IMAGE_URI" + +docker manifest rm "$IMAGE_URI" >/dev/null 2>&1 || true +if docker manifest add --help >/dev/null 2>&1; then + docker manifest create "$IMAGE_URI" + docker manifest add "$IMAGE_URI" "$AMD64_IMAGE_URI" + docker manifest add "$IMAGE_URI" "$ARM64_IMAGE_URI" + docker manifest push --all "$IMAGE_URI" "docker://${IMAGE_URI}" +else + docker manifest create "$IMAGE_URI" "$AMD64_IMAGE_URI" "$ARM64_IMAGE_URI" + docker manifest annotate "$IMAGE_URI" "$AMD64_IMAGE_URI" --os linux --arch amd64 + docker manifest annotate "$IMAGE_URI" "$ARM64_IMAGE_URI" --os linux --arch arm64 + docker manifest push "$IMAGE_URI" +fi + +printf "%s\n" "$IMAGE_TAG" > image-version.txt +printf "Built and pushed %s\n" "$IMAGE_URI" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/package-push-chart.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/package-push-chart.sh new file mode 100755 index 000000000..39c7c3e4a --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/package-push-chart.sh @@ -0,0 +1,194 @@ +#!/bin/bash +set -euo pipefail + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" >&2 + exit 1 + } +} + +validate_ocid() { + local value="$1" + [[ "$value" =~ ^ocid1\.[a-z0-9_-]+\.[a-z0-9_-]*\.[a-z0-9-]*\.[a-z0-9]+$ ]] +} + +validate_repo_path() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$ ]] +} + +validate_chart_path() { + local value="$1" + [[ "$value" = "." || "$value" =~ ^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$ ]] +} + +validate_values_path() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9]+([._-][A-Za-z0-9]+)*(/[A-Za-z0-9]+([._-][A-Za-z0-9]+)*)*\.ya?ml$ ]] +} + +validate_name() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +validate_version() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._:+-]+$ ]] +} + +validate_region_key() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+$ ]] +} + +validate_namespace() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +unset -v COMPARTMENT_ID +unset -v OCIR_REGION_KEY +unset -v CHART_NAME +unset -v CHART_VERSION +unset -v CHART_PATH +unset -v OCIR_REPOSITORY_PATH +unset -v TENANCY_NAMESPACE +unset -v VALUES_FILE + +while getopts c:k:n:v:d:p:t:f: flag; do + case "${flag}" in + c) COMPARTMENT_ID=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + n) CHART_NAME=${OPTARG} ;; + v) CHART_VERSION=${OPTARG} ;; + d) CHART_PATH=${OPTARG} ;; + p) OCIR_REPOSITORY_PATH=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + f) VALUES_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${COMPARTMENT_ID:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${CHART_NAME:-}" ] || [ -z "${CHART_VERSION:-}" ] || [ -z "${CHART_PATH:-}" ] || [ -z "${OCIR_REPOSITORY_PATH:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +require_cmd oci +require_cmd helm +require_cmd python3 + +if ! validate_ocid "$COMPARTMENT_ID"; then + echo "Invalid compartment OCID" >&2 + exit 1 +fi + +if ! validate_region_key "$OCIR_REGION_KEY"; then + echo "Invalid OCIR region key: ${OCIR_REGION_KEY}" >&2 + exit 1 +fi + +if ! validate_name "$CHART_NAME"; then + echo "Invalid chart name: ${CHART_NAME}" >&2 + exit 1 +fi + +if ! validate_version "$CHART_VERSION"; then + echo "Invalid chart version: ${CHART_VERSION}" >&2 + exit 1 +fi + +if ! validate_chart_path "$CHART_PATH"; then + echo "Invalid chart path: ${CHART_PATH}. Use a safe relative chart path." >&2 + exit 1 +fi + +if ! validate_repo_path "$OCIR_REPOSITORY_PATH"; then + echo "Invalid OCIR repository path: ${OCIR_REPOSITORY_PATH}" >&2 + exit 1 +fi + +if [ -n "${TENANCY_NAMESPACE:-}" ] && ! validate_namespace "$TENANCY_NAMESPACE"; then + echo "Invalid tenancy namespace: ${TENANCY_NAMESPACE}" >&2 + exit 1 +fi + +if [ ! -f "${CHART_PATH}/Chart.yaml" ]; then + echo "Chart.yaml not found at ${CHART_PATH}" >&2 + exit 1 +fi + +if [ -n "${VALUES_FILE:-}" ]; then + if ! validate_values_path "$VALUES_FILE"; then + echo "Invalid values file path: ${VALUES_FILE}. Use a safe relative YAML path." >&2 + exit 1 + fi + + if [ ! -f "$VALUES_FILE" ]; then + echo "Values file not found at ${VALUES_FILE}" >&2 + exit 1 + fi + + cp "$VALUES_FILE" "${CHART_PATH}/values.yaml" +fi + +CHART_ARCHIVE="${CHART_NAME}-${CHART_VERSION}.tgz" +cleanup() { + rm -f "$CHART_ARCHIVE" +} +trap cleanup EXIT + +repo_namespace="${TENANCY_NAMESPACE:-}" +if [ -z "$repo_namespace" ]; then + repo_namespace=$(oci artifacts container configuration get --compartment-id "$COMPARTMENT_ID" --query "data.namespace" --raw-output) +fi + +if ! repo_id=$(oci artifacts container repository list --compartment-id "$COMPARTMENT_ID" --display-name "$OCIR_REPOSITORY_PATH/$CHART_NAME" --limit 1 --query "data.items[0].id" --raw-output 2>/dev/null); then + repo_id="" +fi + +if [ "$repo_id" = "null" ]; then + repo_id="" +fi + +if [ -z "$repo_id" ]; then + oci artifacts container repository create --display-name "$OCIR_REPOSITORY_PATH/$CHART_NAME" --compartment-id "$COMPARTMENT_ID" >/dev/null +fi + +repo_path="${OCIR_REGION_KEY}.ocir.io/${repo_namespace}/${OCIR_REPOSITORY_PATH}" + +CHART_PATH="$CHART_PATH" CHART_VERSION="$CHART_VERSION" python3 - <<'PY' +import os +from pathlib import Path + +p = Path(os.environ["CHART_PATH"]) / "Chart.yaml" +text = p.read_text() +lines = [] +seen = False +for line in text.splitlines(): + if line.startswith("version:"): + lines.append(f"version: {os.environ['CHART_VERSION']}") + seen = True + else: + lines.append(line) +if not seen: + lines.append(f"version: {os.environ['CHART_VERSION']}") +p.write_text("\n".join(lines) + "\n") +PY + +helm lint "${CHART_PATH}" +helm package "${CHART_PATH}" + +TOKEN="$(oci raw-request --http-method GET --target-uri "https://${OCIR_REGION_KEY}.ocir.io/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" +if [ -z "$TOKEN" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 +fi + +echo "$TOKEN" | helm registry login "${OCIR_REGION_KEY}.ocir.io" -u BEARER_TOKEN --password-stdin +helm push "$CHART_ARCHIVE" "oci://${repo_path}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-application-metadata.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-application-metadata.sh new file mode 100644 index 000000000..c7347a898 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-application-metadata.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +validate_name() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +unset -v application_env_file +unset -v output_env_file + +while getopts f:o: flag; do + case "${flag}" in + f) application_env_file=${OPTARG} ;; + o) output_env_file=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${application_env_file:-}" ]; then + echo "Missing application metadata file parameter" >&2 + exit 1 +fi + +if [ ! -f "$application_env_file" ]; then + echo "Application metadata file not found at ${application_env_file}" >&2 + exit 1 +fi + +component_name="" + +while IFS= read -r line || [ -n "$line" ]; do + line="${line%$'\r'}" + case "$line" in + ""|\#*) continue ;; + esac + + if [[ ! "$line" =~ ^[a-z_]+=[A-Za-z0-9._:/+-]+$ ]]; then + echo "Invalid line in ${application_env_file}: ${line}" >&2 + exit 1 + fi + + key="${line%%=*}" + value="${line#*=}" + + case "$key" in + component_name) component_name="$value" ;; + *) + echo "Unsupported key in ${application_env_file}: ${key}" >&2 + exit 1 + ;; + esac +done < "$application_env_file" + +if ! validate_name "$component_name"; then + echo "Invalid component_name: ${component_name}" >&2 + exit 1 +fi + +if [ -n "${output_env_file:-}" ]; then + { + printf "component_name=%s\n" "$component_name" + } > "$output_env_file" +fi + +printf "Loaded component metadata for %s\n" "$component_name" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-chart-version.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-chart-version.sh new file mode 100644 index 000000000..cfc0040ef --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/read-chart-version.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +unset -v CHART_PATH + +while getopts p: flag; do + case "${flag}" in + p) CHART_PATH=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${CHART_PATH:-}" ]; then + echo "Missing chart path parameter" >&2 + exit 1 +fi + +if [ ! -f "${CHART_PATH}/Chart.yaml" ]; then + echo "Chart.yaml not found at ${CHART_PATH}" >&2 + exit 1 +fi + +python3 - "$CHART_PATH/Chart.yaml" <<'PY' +import sys +from pathlib import Path + +for line in Path(sys.argv[1]).read_text().splitlines(): + if line.startswith("version:"): + version = line.split(":", 1)[1].strip().strip("\"'") + if version: + print(version) + sys.exit(0) + +print("Chart.yaml is missing a top-level version", file=sys.stderr) +sys.exit(1) +PY diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-image-tag.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-image-tag.sh new file mode 100644 index 000000000..acffc5bcc --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-image-tag.sh @@ -0,0 +1,226 @@ +#!/bin/bash +set -euo pipefail + +SKOPEO_IMAGE="${SKOPEO_IMAGE:-quay.io/skopeo/stable:v1.15.2}" + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Missing dependency: $1" >&2 + exit 1 + } +} + +validate_ocid() { + local value="$1" + [[ "$value" =~ ^ocid1\.[a-z0-9_-]+\.[a-z0-9_-]*\.[a-z0-9-]*\.[a-z0-9]+$ ]] +} + +validate_repo_path() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$ ]] +} + +validate_name() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +validate_release_tag() { + local value="$1" + [[ "$value" =~ ^[0-9]+[.][0-9]+[.][0-9]+-rc[.][0-9]+$ ]] +} + +validate_commit_id() { + local value="$1" + [[ "$value" =~ ^[a-f0-9]{40}$ ]] +} + +validate_region_key() { + local value="$1" + [[ "$value" =~ ^[a-z0-9]+$ ]] +} + +validate_namespace() { + local value="$1" + [[ "$value" =~ ^[A-Za-z0-9._-]+$ ]] +} + +run_skopeo() { + # The build image has Docker; skopeo runs as a pinned container to avoid local installs. + docker run --rm "$SKOPEO_IMAGE" "$@" +} + +image_exists() { + local image_ref="$1" + run_skopeo inspect --creds "BEARER_TOKEN:${TOKEN}" "docker://${image_ref}" >/dev/null 2>&1 +} + +release_tag_exists() { + local object_id + object_id="$(oci devops repository list-refs \ + --repository-id "$SOURCE_REPOSITORY_ID" \ + --ref-type TAG \ + --ref-name "$RELEASE_TAG" \ + --query 'data.items[0]."object-id"' \ + --raw-output)" + [ -n "$object_id" ] && [ "$object_id" != "null" ] +} + +inspect_digest() { + local image_ref="$1" + local inspect_file="$2" + run_skopeo inspect --creds "BEARER_TOKEN:${TOKEN}" "docker://${image_ref}" > "$inspect_file" + python3 -c 'import json, sys; print(json.load(open(sys.argv[1])).get("Digest", ""))' "$inspect_file" +} + +unset -v COMPARTMENT_ID +unset -v OCIR_REGION_KEY +unset -v COMPONENT_NAME +unset -v OCIR_REPOSITORY_PATH +unset -v TENANCY_NAMESPACE +unset -v RELEASE_TAG +unset -v COMMIT_ID +unset -v SOURCE_REPOSITORY_ID +unset -v OUTPUT_ENV_FILE + +while getopts c:k:n:p:t:g:i:r:o: flag; do + case "${flag}" in + c) COMPARTMENT_ID=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + n) COMPONENT_NAME=${OPTARG} ;; + p) OCIR_REPOSITORY_PATH=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + g) RELEASE_TAG=${OPTARG} ;; + i) COMMIT_ID=${OPTARG} ;; + r) SOURCE_REPOSITORY_ID=${OPTARG} ;; + o) OUTPUT_ENV_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${COMPARTMENT_ID:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${COMPONENT_NAME:-}" ] || [ -z "${OCIR_REPOSITORY_PATH:-}" ] || [ -z "${RELEASE_TAG:-}" ] || [ -z "${COMMIT_ID:-}" ] || [ -z "${SOURCE_REPOSITORY_ID:-}" ]; then + echo "Missing required parameters" >&2 + exit 1 +fi + +require_cmd docker +require_cmd oci +require_cmd python3 + +COMMIT_ID="$(printf "%s" "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" + +if ! validate_ocid "$COMPARTMENT_ID"; then + echo "Invalid compartment OCID" >&2 + exit 1 +fi + +if ! validate_ocid "$SOURCE_REPOSITORY_ID"; then + echo "Invalid source repository OCID" >&2 + exit 1 +fi + +if ! validate_region_key "$OCIR_REGION_KEY"; then + echo "Invalid OCIR region key: ${OCIR_REGION_KEY}" >&2 + exit 1 +fi + +if ! validate_name "$COMPONENT_NAME"; then + echo "Invalid component name: ${COMPONENT_NAME}" >&2 + exit 1 +fi + +if ! validate_repo_path "$OCIR_REPOSITORY_PATH"; then + echo "Invalid OCIR repository path: ${OCIR_REPOSITORY_PATH}" >&2 + exit 1 +fi + +if ! validate_release_tag "$RELEASE_TAG"; then + echo "Invalid release tag: ${RELEASE_TAG}" >&2 + exit 1 +fi + +if ! validate_commit_id "$COMMIT_ID"; then + echo "Invalid commit id: ${COMMIT_ID}" >&2 + exit 1 +fi + +if [ -n "${TENANCY_NAMESPACE:-}" ] && ! validate_namespace "$TENANCY_NAMESPACE"; then + echo "Invalid tenancy namespace: ${TENANCY_NAMESPACE}" >&2 + exit 1 +fi + +repo_namespace="${TENANCY_NAMESPACE:-}" +if [ -z "$repo_namespace" ]; then + repo_namespace=$(oci artifacts container configuration get --compartment-id "$COMPARTMENT_ID" --query "data.namespace" --raw-output) +fi + +REMOTE_REGISTRY="${OCIR_REGION_KEY}.ocir.io" +IMAGE_REPOSITORY="${OCIR_REPOSITORY_PATH%/}" +IMAGE_REPOSITORY_REF="${REMOTE_REGISTRY}/${repo_namespace}/${IMAGE_REPOSITORY}" +SOURCE_SHA_TAG="${COMMIT_ID:0:7}" +SOURCE_IMAGE="${IMAGE_REPOSITORY_REF}:${SOURCE_SHA_TAG}" +TARGET_IMAGE="${IMAGE_REPOSITORY_REF}:${RELEASE_TAG}" + +TOKEN="$(oci raw-request --http-method GET --target-uri "https://${REMOTE_REGISTRY}/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" +if [ -z "$TOKEN" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 +fi + +if ! image_exists "$SOURCE_IMAGE"; then + echo "Source image does not exist: ${SOURCE_IMAGE}" >&2 + exit 1 +fi + +if image_exists "$TARGET_IMAGE"; then + echo "Target image already exists: ${TARGET_IMAGE}" >&2 + exit 1 +fi + +if release_tag_exists; then + echo "Release Git tag already exists: ${RELEASE_TAG}" >&2 + exit 1 +fi + +source_inspect="$(mktemp)" +target_inspect="$(mktemp)" +source_digest="$(inspect_digest "$SOURCE_IMAGE" "$source_inspect")" + +run_skopeo copy \ + --all \ + --src-creds "BEARER_TOKEN:${TOKEN}" \ + --dest-creds "BEARER_TOKEN:${TOKEN}" \ + "docker://${SOURCE_IMAGE}" \ + "docker://${TARGET_IMAGE}" + +# Digest equality proves the release tag points at the exact image built from the commit SHA. +target_digest="$(inspect_digest "$TARGET_IMAGE" "$target_inspect")" +if [ "$source_digest" != "$target_digest" ]; then + echo "Released image digest mismatch: ${source_digest} != ${target_digest}" >&2 + exit 1 +fi + +if release_tag_exists; then + echo "Release Git tag already exists: ${RELEASE_TAG}" >&2 + exit 1 +fi + +oci devops repository create-or-update-git-tag-details \ + --repository-id "$SOURCE_REPOSITORY_ID" \ + --ref-name "$RELEASE_TAG" \ + --object-id "$COMMIT_ID" >/dev/null + +if [ -n "${OUTPUT_ENV_FILE:-}" ]; then + { + printf "source_sha_tag=%s\n" "$SOURCE_SHA_TAG" + printf "promoted_image_tag=%s\n" "$RELEASE_TAG" + printf "promoted_image_uri=%s\n" "$TARGET_IMAGE" + } > "$OUTPUT_ENV_FILE" +fi + +printf "Tagged repository commit %s as %s\n" "$COMMIT_ID" "$RELEASE_TAG" +printf "Released %s to %s\n" "$SOURCE_IMAGE" "$TARGET_IMAGE" +printf "Digest: %s\n" "$target_digest" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-print-summary.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-print-summary.sh new file mode 100644 index 000000000..8b90a9dba --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-print-summary.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +unset -v RELEASE_ENV_FILE +unset -v PROMOTION_ENV_FILE + +while getopts e:p: flag; do + case "${flag}" in + e) RELEASE_ENV_FILE=${OPTARG} ;; + p) PROMOTION_ENV_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${RELEASE_ENV_FILE:-}" ] || [ -z "${PROMOTION_ENV_FILE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +source "$RELEASE_ENV_FILE" +source "$PROMOTION_ENV_FILE" + +printf "Release tag: %s\n" "$release_tag" +printf "Source commit: %s\n" "$resolved_commit_id" +printf "Source image tag: %s\n" "$source_sha_tag" +printf "Chart version: %s\n" "$chart_version" +printf "Released image: %s\n" "$promoted_image_uri" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-resolve-inputs.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-resolve-inputs.sh new file mode 100644 index 000000000..b7743d161 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-resolve-inputs.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -euo pipefail + +unset -v RELEASE_TAG +unset -v COMMIT_ID +unset -v SOURCE_DIR +unset -v CHART_SOURCE_DIR +unset -v CHART_PATH +unset -v CHART_REPO_PREFIX +unset -v OUTPUT_ENV_FILE + +while getopts g:i:s:a:p:r:o: flag; do + case "${flag}" in + g) RELEASE_TAG=${OPTARG} ;; + i) COMMIT_ID=${OPTARG} ;; + s) SOURCE_DIR=${OPTARG} ;; + a) CHART_SOURCE_DIR=${OPTARG} ;; + p) CHART_PATH=${OPTARG} ;; + r) CHART_REPO_PREFIX=${OPTARG} ;; + o) OUTPUT_ENV_FILE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${RELEASE_TAG:-}" ] || [ -z "${CHART_SOURCE_DIR:-}" ] || [ -z "${CHART_PATH:-}" ] || [ -z "${CHART_REPO_PREFIX:-}" ] || [ -z "${OUTPUT_ENV_FILE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" +: "${OCI_STAGE_ID:?OCI_STAGE_ID is required}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APPLICATION_ENV_FILE=".oci-devops/application.env" +APPLICATION_VALIDATED_ENV_FILE=".oci-devops/application.validated.env" + +find_application_sources() { + local candidate + for candidate in "${OCI_WORKSPACE_DIR}"/*; do + [ -d "$candidate" ] || continue + [ -f "${candidate}/.oci-devops/application.env" ] || continue + basename "$candidate" + done +} + +application_sources=() +while IFS= read -r source_name; do + application_sources+=("$source_name") +done < <(find_application_sources) + +if [ -z "${SOURCE_DIR:-}" ]; then + if [ -z "${COMMIT_ID:-}" ] || [ "$COMMIT_ID" = "CHANGE_ME" ]; then + if [ "${#application_sources[@]}" -ne 1 ]; then + echo "Unable to select one application source for release. Found: ${application_sources[*]:-none}" >&2 + exit 1 + fi + SOURCE_DIR="${application_sources[0]}" + else + normalized_commit_id="$(printf "%s" "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" + for source_candidate in "${application_sources[@]}"; do + if git -C "${OCI_WORKSPACE_DIR}/${source_candidate}" cat-file -e "${normalized_commit_id}^{commit}" 2>/dev/null; then + SOURCE_DIR="$source_candidate" + break + fi + done + if [ -z "${SOURCE_DIR:-}" ]; then + echo "Unable to find application source containing commit: ${normalized_commit_id}" >&2 + exit 1 + fi + fi +fi + +cd "${OCI_WORKSPACE_DIR}/${SOURCE_DIR}" +bash "${SCRIPT_DIR}/read-application-metadata.sh" \ + -f "$APPLICATION_ENV_FILE" \ + -o "$APPLICATION_VALIDATED_ENV_FILE" +source "$APPLICATION_VALIDATED_ENV_FILE" + +source_repository_id="$(oci devops build-pipeline-stage get --stage-id "$OCI_STAGE_ID" --query 'data."build-source-collection".items[?name==`'"$SOURCE_DIR"'`]."repository-id" | [0]' --raw-output)" +if [ -z "$source_repository_id" ] || [ "$source_repository_id" = "null" ]; then + echo "Unable to derive source repository id for build source: ${SOURCE_DIR}" >&2 + exit 1 +fi + +if [ -z "${COMMIT_ID:-}" ] || [ "$COMMIT_ID" = "CHANGE_ME" ]; then + resolved_commit_id="$(git rev-parse origin/main)" +else + resolved_commit_id="$COMMIT_ID" +fi +resolved_commit_id="$(printf "%s" "$resolved_commit_id" | tr '[:upper:]' '[:lower:]')" + +git merge-base --is-ancestor "$resolved_commit_id" origin/main + +{ + printf "release_tag=%q\n" "$RELEASE_TAG" + printf "resolved_commit_id=%q\n" "$resolved_commit_id" + printf "source_repository_id=%q\n" "$source_repository_id" + printf "source_dir=%q\n" "$SOURCE_DIR" + printf "chart_source_dir=%q\n" "$CHART_SOURCE_DIR" + printf "chart_path=%q\n" "$CHART_PATH" + printf "chart_name=%q\n" "$(basename "$CHART_PATH")" + printf "chart_repo_prefix=%q\n" "$CHART_REPO_PREFIX" + printf "component_name=%q\n" "$component_name" + printf "release_env_file=%q\n" "application-release.env" + printf "chart_version=%q\n" "" +} > "$OUTPUT_ENV_FILE" + +printf "Release tag: %s\n" "$RELEASE_TAG" +printf "Source commit: %s\n" "$resolved_commit_id" +printf "Component: %s\n" "$component_name" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-verify-chart.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-verify-chart.sh new file mode 100644 index 000000000..54d29b84b --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/repos/pipelines/script/release-verify-chart.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -euo pipefail + +unset -v RELEASE_ENV_FILE +unset -v OCIR_REGION_KEY +unset -v TENANCY_NAMESPACE + +while getopts e:k:t: flag; do + case "${flag}" in + e) RELEASE_ENV_FILE=${OPTARG} ;; + k) OCIR_REGION_KEY=${OPTARG} ;; + t) TENANCY_NAMESPACE=${OPTARG} ;; + *) + echo "Error in command line parsing" >&2 + exit 1 + ;; + esac +done + +if [ -z "${RELEASE_ENV_FILE:-}" ] || [ -z "${OCIR_REGION_KEY:-}" ] || [ -z "${TENANCY_NAMESPACE:-}" ]; then + echo "Missing parameters" >&2 + exit 1 +fi + +: "${OCI_WORKSPACE_DIR:?OCI_WORKSPACE_DIR is required}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$RELEASE_ENV_FILE" + +cd "${OCI_WORKSPACE_DIR}/${chart_source_dir}" +chart_version="$(bash "${SCRIPT_DIR}/read-chart-version.sh" -p "$chart_path")" + +remote_registry="${OCIR_REGION_KEY}.ocir.io" +chart_url="oci://${remote_registry}/${TENANCY_NAMESPACE}/${chart_repo_prefix%/}/${chart_name}" + +# Helm uses an OCIR bearer token here because release validation is registry-only. +token="$(oci raw-request --http-method GET --target-uri "https://${remote_registry}/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" +if [ -z "$token" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 +fi + +echo "$token" | helm registry login "$remote_registry" -u BEARER_TOKEN --password-stdin +if ! helm show chart "$chart_url" --version "$chart_version" >/dev/null 2>&1; then + echo "Application chart is missing: ${chart_url}:${chart_version}" >&2 + exit 1 +fi + +{ + grep -v '^chart_version=' "$RELEASE_ENV_FILE" || true + printf "chart_version=%q\n" "$chart_version" +} > "${RELEASE_ENV_FILE}.tmp" +mv "${RELEASE_ENV_FILE}.tmp" "$RELEASE_ENV_FILE" + +printf "Verified application chart: %s:%s\n" "$chart_url" "$chart_version" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/schema.yaml b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/schema.yaml new file mode 100644 index 000000000..a9d3ddf7d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/schema.yaml @@ -0,0 +1,621 @@ +title: OKE Helm Starter Stack +description: Bootstrap an existing OKE cluster for Helm delivery through OCI DevOps. +stackDescription: Creates OCI DevOps repositories, build and deployment pipelines, and application namespaces for OKE. +schemaVersion: 1.1.0 +version: "20260703" +locale: en +informationalText: "Start with the Pipelines Repository button and the Next Steps output. Initialize each application namespace, deploy its baseline chart, then develop components through pull requests, builds, and release promotion." + +variableGroups: + - title: Hidden + visible: false + variables: + - tenancy_ocid + - region + - current_user_ocid + - compartment_ocid + - development_mode + + - title: OCI DevOps + variables: + - devops_compartment_id + - devops_project_name + - devops_project_description + - auth_token + - create_notification_topic + - notification_topic_id + - notification_topic_name + - notification_topic_description + - devops_log_is_enabled + - devops_log_group_name + - devops_log_group_description + - devops_log_name + - devops_log_retention_period_in_days + - namespace_init_secret_name + + - title: Applications + variables: + - applications + - application_namespace_help + + - title: OKE Pre-Prod Environment + variables: + - oke_compartment_id + - oke_cluster_id + - network_compartment_id + - oke_vcn_id + - oke_worker_subnet_id + - oke_worker_nsg_id + + - title: OKE Prod Environment + variables: + - prod_oke_compartment_id + - prod_oke_cluster_id + - prod_network_compartment_id + - prod_oke_vcn_id + - prod_oke_worker_subnet_id + - prod_oke_worker_nsg_id + + - title: IAM + variables: + - create_iam + - iam_domain_compartment_id + - devops_iam_domain_id + - namespace_init_secret_compartment_id + - devops_dynamic_group_name + - devops_policy_name + + - title: Cluster Administration (Optional) + variables: + - enable_cluster_admin + - cluster_admin_artifact_repository_name + - cluster_administration_help + - cluster_administration + +variables: + devops_compartment_id: + title: DevOps Compartment + description: Compartment where this stack creates OCI DevOps resources and OCIR repositories. Defaults to the stack compartment. + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + + devops_project_name: + title: DevOps Project Name + description: Name of the OCI DevOps project. + type: string + default: oke-devops-starter + pattern: ".*\\S.*" + minLength: 1 + maxLength: 100 + required: true + + devops_project_description: + title: DevOps Project Description + type: string + default: Beginner-friendly OCI DevOps project for OKE Helm delivery + + devops_log_group_name: + title: Log Group Name + type: string + default: oke-helm-starter-logs + required: true + visible: ${devops_log_is_enabled} + + devops_log_group_description: + title: Log Group Description + type: string + default: OCI DevOps logs for the OKE Helm starter project + visible: ${devops_log_is_enabled} + + devops_log_name: + title: DevOps Service Log Name + description: Optional name for the OCI Logging service log that captures DevOps project logs. Leave empty to derive it from the log group and project name. + type: string + default: "" + visible: ${devops_log_is_enabled} + + devops_log_is_enabled: + title: Enable DevOps Service Log + description: Create an OCI Logging log group and service log for this DevOps project. + type: boolean + default: true + + devops_log_retention_period_in_days: + title: Log Retention + description: Number of days to retain OCI DevOps logs. + type: integer + minimum: 30 + maximum: 180 + multipleOf: 30 + default: 30 + required: true + visible: ${devops_log_is_enabled} + + create_notification_topic: + title: Create Notification Topic + description: Create an OCI Notifications topic for the DevOps project. + type: boolean + default: true + + notification_topic_id: + title: Existing Notification Topic OCID + description: Existing topic used when a new topic is not created. + type: string + required: true + visible: + not: + - ${create_notification_topic} + + notification_topic_name: + title: Notification Topic Name + type: string + default: oke-helm-starter-topic + required: true + visible: ${create_notification_topic} + + notification_topic_description: + title: Notification Topic Description + type: string + default: OCI DevOps notifications for the OKE Helm starter project + visible: ${create_notification_topic} + + oke_compartment_id: + title: OKE Pre-Prod Compartment + description: Compartment containing the existing pre-prod OKE cluster. + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + + oke_cluster_id: + title: OKE Pre-Prod Cluster + description: Existing pre-prod OKE cluster that receives development and staging Helm deployments. + type: oci:container:cluster:id + required: true + dependsOn: + compartmentId: ${oke_compartment_id} + + prod_oke_cluster_id: + title: OKE Prod Cluster + description: Existing prod OKE cluster that receives production Helm deployments. + type: oci:container:cluster:id + required: true + dependsOn: + compartmentId: ${prod_oke_compartment_id} + + prod_oke_compartment_id: + title: OKE Prod Compartment + description: Compartment containing the existing prod OKE cluster. + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + + prod_network_compartment_id: + title: OKE Prod Network Compartment + description: Network compartment containing the prod worker subnet and optional NSG. + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + + prod_oke_vcn_id: + title: OKE Prod VCN + description: VCN containing the worker subnet used for OCI DevOps private endpoint access to the prod OKE cluster. + type: oci:core:vcn:id + required: true + dependsOn: + compartmentId: ${prod_network_compartment_id} + + network_compartment_id: + title: OKE Pre-Prod Network Compartment + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + + oke_vcn_id: + title: OKE Pre-Prod VCN + description: VCN containing the worker subnet used for OCI DevOps private endpoint access to the pre-prod OKE cluster. + type: oci:core:vcn:id + required: true + dependsOn: + compartmentId: ${network_compartment_id} + + oke_worker_subnet_id: + title: OKE Pre-Prod Worker Subnet + description: Worker subnet used by OCI DevOps private endpoint connections and shell stages. It must reach the Kubernetes private endpoint. + type: oci:core:subnet:id + required: true + dependsOn: + compartmentId: ${network_compartment_id} + vcnId: ${oke_vcn_id} + + oke_worker_nsg_id: + title: OKE Pre-Prod Worker NSG + description: Optional worker NSG attached to OCI DevOps private endpoint connections and shell stage container networking. + type: oci:core:nsg:id + required: false + dependsOn: + compartmentId: ${network_compartment_id} + vcnId: ${oke_vcn_id} + + prod_oke_worker_subnet_id: + title: OKE Prod Worker Subnet + description: Worker subnet used by OCI DevOps private endpoint connections and shell stages for production. + type: oci:core:subnet:id + required: true + dependsOn: + compartmentId: ${prod_network_compartment_id} + vcnId: ${prod_oke_vcn_id} + + prod_oke_worker_nsg_id: + title: OKE Prod Worker NSG + description: Optional worker NSG for production private endpoint access and shell stages. + type: oci:core:nsg:id + required: false + dependsOn: + compartmentId: ${prod_network_compartment_id} + vcnId: ${prod_oke_vcn_id} + + application_namespace_help: + title: How Applications Work + type: text + multiline: true + default: | + Configure one or more applications. Each application gets a chart repository and baseline deployment pipeline. Each globally unique component gets its own source repository, build pipeline, PR pipeline, release pipeline, and component deployment pipelines. + + Example: + [ + { + "name": "orders", + "components": [ + { + "name": "orders-api", + "build_spec_path": "java/java-build-pipeline.yaml" + }, + { + "name": "orders-worker", + "build_spec_path": "java/java-build-pipeline.yaml" + } + ] + }, + { + "name": "payments", + "components": [ + { "name": "payments-api" } + ] + } + ] + + The optional build_spec_path is relative to the pipelines repository. When omitted, the stack generates -build-pipeline.yaml. When specified, the stack creates the file and any parent folders from the default template only if it is missing. After that first commit the specification is user-owned and is never refreshed or overwritten. Multiple components can reference the same specification. + + applications: + title: Applications + description: "JSON array of applications and globally unique components to bootstrap. Components may set build_spec_path to reference a user-owned specification in the pipelines repository; omitted paths use the generated -build-pipeline.yaml convention." + type: text + multiline: true + pattern: "^\\s*\\[[\\s\\S]*\\]\\s*$" + minLength: 2 + maxLength: 8192 + required: true + default: | + [ + { + "name": "sample-app", + "components": [ + { + "name": "sample-api" + }, + { + "name": "sample-worker" + } + ] + } + ] + + enable_cluster_admin: + title: Enable Cluster Administration + description: Create the optional cluster-admin repository, immutable values repository, validation and mirror builds, and noprod/prod orchestration pipelines. + type: boolean + default: false + + cluster_admin_artifact_repository_name: + title: Cluster Administration Artifact Repository Name + description: Optional Generic Artifact repository display name for immutable tool values and cluster deployment plans. Leave empty to derive -cluster-admin-values. + type: string + default: "" + pattern: "^$|^[A-Za-z0-9]([A-Za-z0-9._ -]*[A-Za-z0-9._-])?$" + maxLength: 255 + visible: ${enable_cluster_admin} + + development_mode: + title: Development Mode + description: Internal packaging mode used only by the maintained development stack. + type: boolean + default: false + visible: false + + cluster_administration_help: + title: How Cluster Administration Works + type: text + multiline: true + visible: ${enable_cluster_admin} + default: | + Configure one shared Kubernetes tool topology and its pinned public Helm chart sources for the existing noprod and prod OKE clusters. Each cluster gets one orchestrator; prod adds an approval before execution. Values, supplemental resources, and cluster-wide manifests remain separate in Git. Independent tools run in parallel; depends_on places a tool after its prerequisites. + + Public charts are pinned in the generated cluster-admin repository and mirrored into OCIR. Pull requests validate configuration without cluster access. After merge, the build publishes immutable values and runs only affected stages. Noprod starts immediately; prod requires approval before any selected mutation. + + Traditional Helm repository example: + { + "name": "keda", + "repository": "https://kedacore.github.io/charts", + "chart": "keda", + "version": "2.20.1", + "namespace": "keda", + "depends_on": [] + } + + OCI chart repository example (the chart name is appended to the repository path): + { + "name": "external-dns", + "repository": "oci://registry-1.docker.io/bitnamicharts", + "chart": "external-dns", + "version": "9.0.3", + "namespace": "external-dns", + "depends_on": [] + } + + cluster_administration: + title: Cluster Administration + description: JSON configuration for the shared Kubernetes tool topology. Repository accepts an HTTPS Helm repository or an OCI base path; chart is appended to OCI paths. Every tool requires a pinned version. Namespaces default to tool names. + type: text + multiline: true + pattern: "^\\s*\\{[\\s\\S]*\\}\\s*$" + minLength: 2 + maxLength: 8192 + required: true + visible: ${enable_cluster_admin} + default: | + { + "tools": [ + { + "name": "keda", + "repository": "https://kedacore.github.io/charts", + "chart": "keda", + "version": "2.20.1", + "namespace": "keda", + "depends_on": [] + }, + { + "name": "kube-prometheus", + "repository": "https://prometheus-community.github.io/helm-charts", + "chart": "kube-prometheus-stack", + "version": "87.10.1", + "namespace": "monitoring", + "depends_on": [] + } + ] + } + + namespace_init_secret_name: + title: Application Bootstrap Pull Secret Name + description: Docker-registry secret created by each application's noprod and prod bootstrap stages. + type: string + default: ocirsecret + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + minLength: 1 + maxLength: 63 + required: true + + auth_token: + title: Auth Token + description: Auth token for the current OCI user. Resource Manager uses it once to seed the hosted DevOps repositories. + type: password + required: true + + create_iam: + title: Create DevOps IAM Policies And Dynamic Group + description: Enable only if you want this stack to create the DevOps dynamic group and policy. Leave off when IAM has already been prepared. + type: boolean + default: false + + iam_domain_compartment_id: + title: IAM Domain Compartment + type: oci:identity:compartment:id + default: ${tenancy_ocid} + required: true + visible: ${create_iam} + + devops_iam_domain_id: + title: OCI DevOps Identity Domain + type: oci:identity:domains:id + required: true + visible: ${create_iam} + dependsOn: + compartmentId: ${iam_domain_compartment_id} + + namespace_init_secret_compartment_id: + title: Vault Compartment + description: Compartment containing the Vault secret used as the OCIR pull password during application bootstrap. + type: oci:identity:compartment:id + default: ${compartment_ocid} + required: true + visible: ${create_iam} + + devops_dynamic_group_name: + title: DevOps Dynamic Group Name + type: string + default: OkeHelmStarterDevOpsDynamicGroup + required: true + visible: ${create_iam} + + devops_policy_name: + title: DevOps Policy Name + type: string + default: oke-helm-starter-devops-policy + required: true + visible: ${create_iam} + +outputs: + devops_project_id: + title: DevOps Project + description: OCI DevOps project created by this stack. + type: ocid + + pipelines_repository_url: + title: Pipelines Repository + description: Shared build specifications and helper scripts. + type: link + displayText: Open Pipelines Repository + + cluster_admin_repository_url: + title: Cluster Administration Repository + description: Cluster-wide resources, tool catalog, per-cluster Helm values, and supplemental resources. + type: link + displayText: Open Cluster Administration Repository + + cluster_admin_values_repository_id: + title: Cluster Administration Values Repository + description: Generic Artifact repository containing immutable per-commit Helm values and cluster deployment plans. + type: ocid + + cluster_admin_build_pipeline_ids: + title: Cluster Administration Build Pipelines + description: PR validation, chart mirroring, and changed-target dispatch pipelines. + type: map + + cluster_admin_deploy_pipeline_ids: + title: Cluster Administration Deployment Pipelines + description: Noprod and approved prod orchestration pipelines plus explicit decommission pipelines. + type: map + + component_source_repository_urls: + title: Component Source Repositories + description: Component name to source repository URL. + type: map + + application_chart_repository_urls: + title: Application Chart Repositories + description: Application name to chart repository URL. + type: map + + component_build_pipeline_ids: + title: Component Build Pipelines + description: Component name to build pipeline OCID. + type: map + + component_pr_pipeline_ids: + title: Component PR Pipelines + description: Component name to pull request validation pipeline OCID. + type: map + + component_release_build_pipeline_ids: + title: Component Release Build Pipelines + description: Component name to release build pipeline OCID. + type: map + + application_package_pipeline_ids: + title: Application Package Pipelines + description: Application name to baseline chart package pipeline OCID. + type: map + + application_deploy_pipeline_ids: + title: Application Deployment Pipelines + description: Application name to baseline deployment pipeline OCID. + type: map + + application_bootstrap_pipeline_ids: + title: Application Bootstrap Pipelines + description: Application name to the idempotent parallel noprod/prod namespace and pull-secret bootstrap pipeline. + type: map + + component_deploy_pipeline_ids: + title: Component Deployment Pipelines + description: Component and environment to deployment pipeline OCID. + type: map + + oke_noprod_deploy_environment_id: + title: OKE Pre-Prod Deployment Environment + type: ocid + + oke_prod_deploy_environment_id: + title: OKE Prod Deployment Environment + type: ocid + + application_components: + title: Applications And Components + description: Application name to component names. + type: map + + application_namespaces: + title: Pre-Prod Namespaces + type: map + + application_prod_namespaces: + title: Prod Namespaces + type: map + + application_ocir_charts: + title: Application OCIR Charts + type: map + + component_ocir_charts: + title: Component OCIR Charts + type: map + + component_image_repositories: + title: Component Image Repositories + type: map + + estimated_devops_resources: + title: Estimated DevOps Resources + description: Estimated repository, build pipeline, deployment pipeline, and deployment stage counts for service-limit review. + type: map + + next_steps: + title: Next Steps + description: Suggested workflow derived from the configured applications and components. + type: string + +primaryOutputButton: ${pipelines_repository_url} + +outputGroups: + - title: Start Here + outputs: + - ${pipelines_repository_url} + - ${cluster_admin_repository_url} + - ${devops_project_id} + - ${application_components} + - ${application_namespaces} + - ${application_prod_namespaces} + - ${application_bootstrap_pipeline_ids} + - ${estimated_devops_resources} + - ${next_steps} + + - title: Repositories + outputs: + - ${cluster_admin_repository_url} + - ${component_source_repository_urls} + - ${application_chart_repository_urls} + + - title: Build Pipelines + outputs: + - ${cluster_admin_build_pipeline_ids} + - ${component_pr_pipeline_ids} + - ${component_build_pipeline_ids} + - ${component_release_build_pipeline_ids} + - ${application_package_pipeline_ids} + + - title: Deployment Pipelines And Environments + outputs: + - ${cluster_admin_deploy_pipeline_ids} + - ${application_bootstrap_pipeline_ids} + - ${application_deploy_pipeline_ids} + - ${component_deploy_pipeline_ids} + - ${oke_noprod_deploy_environment_id} + - ${oke_prod_deploy_environment_id} + + - title: OCIR + outputs: + - ${cluster_admin_values_repository_id} + - ${application_ocir_charts} + - ${component_ocir_charts} + - ${component_image_repositories} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/seed_repo.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/seed_repo.sh new file mode 100644 index 000000000..fe0f8c1fc --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/seed_repo.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -euo pipefail + +require_env() { + local name="$1" + if [ -z "${!name:-}" ]; then + echo "Missing required environment variable: ${name}" >&2 + exit 1 + fi +} + +require_env REGION +require_env GIT_USERNAME +require_env GIT_PASSWORD +require_env REPO_CLONE_URL +require_env SOURCE_REPO +require_env SEED_MODE +require_env SEED_PATHS + +if [[ ! "$REGION" =~ ^[a-z]{2}(-[a-z]+)*-[a-z]+-[0-9]+$ ]]; then + echo "Invalid OCI region: ${REGION}" >&2 + exit 1 +fi + +if [[ "$REPO_CLONE_URL" != https://devops.scmservice."${REGION}".oci.oraclecloud.com/* ]]; then + echo "Repository URL must be an OCI DevOps HTTPS URL for ${REGION}" >&2 + exit 1 +fi + +if [ "$SEED_MODE" != "empty-repository" ] && [ "$SEED_MODE" != "add-only" ] && [ "$SEED_MODE" != "refresh" ]; then + echo "SEED_MODE must be empty-repository, add-only, or refresh." >&2 + exit 1 +fi + +if [[ "$SOURCE_REPO" = /* ]]; then + SOURCE_REPO=".${SOURCE_REPO}" +fi + +if [[ "$SOURCE_REPO" = *".."* ]] || [ ! -d "$SOURCE_REPO" ]; then + echo "Invalid source repository path: ${SOURCE_REPO}" >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d)" +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +mkdir -p "${WORK_DIR}/home" +export HOME="${WORK_DIR}/home" + +cat >>"${HOME}/.netrc" </dev/null 2>&1; then + echo "Repository already contains a commit; preserving developer-owned content." + exit 0 +fi + +added_paths=0 +if [ -n "${REMOVE_PATHS:-}" ]; then + if [ "$SEED_MODE" != "refresh" ]; then + echo "REMOVE_PATHS is allowed only in refresh mode." >&2 + exit 1 + fi + while IFS= read -r relative_path || [ -n "$relative_path" ]; do + [ -n "$relative_path" ] || continue + if [[ "$relative_path" = /* ]] || [[ "$relative_path" = *".."* ]]; then + echo "Invalid removal path: ${relative_path}" >&2 + exit 1 + fi + destination_path="${WORK_DIR}/repo/${relative_path}" + if [ -e "$destination_path" ]; then + rm -rf "$destination_path" + echo "Removed obsolete template path: ${relative_path}" + added_paths=$((added_paths + 1)) + fi + done <<<"$REMOVE_PATHS" +fi + +while IFS= read -r relative_path || [ -n "$relative_path" ]; do + [ -n "$relative_path" ] || continue + + if [[ "$relative_path" = /* ]] || [[ "$relative_path" = *".."* ]]; then + echo "Invalid seed path: ${relative_path}" >&2 + exit 1 + fi + + source_path="${SOURCE_REPO}/${relative_path}" + destination_path="${WORK_DIR}/repo/${relative_path}" + + if [ ! -e "$source_path" ]; then + echo "Seed source does not exist: ${source_path}" >&2 + exit 1 + fi + + if [ -e "$destination_path" ] && [ "$SEED_MODE" != "refresh" ]; then + echo "Preserving existing path: ${relative_path}" + continue + fi + + mkdir -p "$(dirname "$destination_path")" + if [ "$SEED_MODE" = "refresh" ] && [ -d "$source_path" ]; then + mkdir -p "$destination_path" + cp -a "$source_path/." "$destination_path/" + echo "Refreshed template path: ${relative_path}" + else + cp -a "$source_path" "$destination_path" + echo "$([ "$SEED_MODE" = "refresh" ] && echo Refreshed || echo Added) template path: ${relative_path}" + fi + added_paths=$((added_paths + 1)) +done <<<"$SEED_PATHS" + +if [ "$added_paths" -eq 0 ]; then + echo "No starter paths were added." + exit 0 +fi + +cd "${WORK_DIR}/repo" +git config user.email "resource-manager@oracle.com" +git config user.name "${GIT_USERNAME}" + +git add . + +if git diff --cached --quiet; then + echo "No repository content changes to push." + exit 0 +fi + +git commit -m "$([ "$SEED_MODE" = "refresh" ] && echo Refresh || echo Add) Resource Manager template files" +git push origin HEAD:main diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/update_orm_stack.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/update_orm_stack.sh new file mode 100755 index 000000000..a5a43e504 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/script/update_orm_stack.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +DEFAULT_STACK_ID="ocid1.ormstack.oc1.eu-frankfurt-1.amaaaaaauevftmqahquri6ywkezcpdskdb5vcxskx52k4443h65awnsjwuva" +STACK_ID="${ORM_STACK_ID:-$DEFAULT_STACK_ID}" +IFS="." read -r _ _ _ INFERRED_STACK_REGION _ <<< "$STACK_ID" +STACK_REGION="${ORM_STACK_REGION:-$INFERRED_STACK_REGION}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ZIP_PATH="$ROOT_DIR/stack.zip" + +cd "$ROOT_DIR" +STACK_DEVELOPMENT_MODE="${STACK_DEVELOPMENT_MODE:-true}" ./update.sh + +oci resource-manager stack update \ + --region "$STACK_REGION" \ + --stack-id "$STACK_ID" \ + --config-source "$ZIP_PATH" \ + --force \ + --wait-for-state ACTIVE \ + --query 'data."lifecycle-state"' \ + --raw-output + +printf "Uploaded %s to Resource Manager stack %s in %s\n" "$ZIP_PATH" "$STACK_ID" "$STACK_REGION" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/stack.zip b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/stack.zip new file mode 100644 index 000000000..09ce07195 Binary files /dev/null and b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/stack.zip differ diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-pipeline.yaml.tpl new file mode 100644 index 000000000..df2f07811 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-pipeline.yaml.tpl @@ -0,0 +1,50 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + application_chart_source_dir: "application-chart" + application_chart_path: "${application_chart_path}" + application_chart_repo_prefix: "${application_chart_repo_prefix}" + application_chart_name: "${application_chart_name}" + application_baseline_env_file: "/workspace/application-baseline.env" + repo_compartment_id: "${repo_compartment_id}" + region: "${region}" + region_key: "${region_key}" + tenancy_namespace: "${tenancy_namespace}" + exportedVariables: + - chart_version + +steps: + - type: Command + name: Read application chart version + failImmediatelyOnError: true + command: | + export OCI_CLI_REGION="$${region}" + + chart_version="$(bash "$${OCI_PRIMARY_SOURCE_DIR}/script/read-chart-version.sh" \ + -p "$${OCI_WORKSPACE_DIR}/$${application_chart_source_dir}/$${application_chart_path}")" + printf "chart_version=%q\n" "$${chart_version}" > "$${application_baseline_env_file}" + export chart_version + + printf "Application baseline chart version: %s\n" "$${chart_version}" + + - type: Command + name: Package application chart + failImmediatelyOnError: true + command: | + source "$${application_baseline_env_file}" + export chart_version + + cd "$${OCI_WORKSPACE_DIR}/$${application_chart_source_dir}" + + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/package-push-chart.sh" \ + -c "$${repo_compartment_id}" \ + -k "$${region_key}" \ + -n "$${application_chart_name}" \ + -v "$${chart_version}" \ + -d "$${application_chart_path}" \ + -p "$${application_chart_repo_prefix}" \ + -t "$${tenancy_namespace}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-values.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-values.yaml.tpl new file mode 100644 index 000000000..bdf43cd89 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-baseline-values.yaml.tpl @@ -0,0 +1 @@ +# Cluster-specific shared namespace values belong here. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-bootstrap-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-bootstrap-command-spec.yaml.tpl new file mode 100644 index 000000000..06a40d1e1 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-bootstrap-command-spec.yaml.tpl @@ -0,0 +1,120 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +env: + variables: + kube_endpoint: "${kube_endpoint}" + noprod_cluster_id: "${noprod_oke_cluster_id}" + OCI_CLI_REGION: "${region}" + ocir_registry: "${region_key}.ocir.io" + prod_cluster_id: "${prod_oke_cluster_id}" + target_registry_username: "$${registry_username}" + target_pull_password_secret_ocid: "$${pull_password_secret_ocid}" + target_secret_name: "$${secret_name}" + +steps: + - type: Command + name: Resolve bootstrap target + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + : "$${OCI_STAGE_ID:?OCI_STAGE_ID is required to resolve the bootstrap target}" + + stage_json="$${OCI_WORKSPACE_DIR}/bootstrap-stage.json" + context_file="$${OCI_WORKSPACE_DIR}/bootstrap-context.env" + oci devops deploy-stage get --stage-id "$${OCI_STAGE_ID}" >"$${stage_json}" + python3 - "$${stage_json}" >"$${context_file}" <<'PY' + import json, shlex, sys + + stage = json.load(open(sys.argv[1], encoding="utf-8"))["data"] + tags = stage.get("freeform-tags") or stage.get("free-form-tags") or {} + required = ("application", "cluster", "namespace") + missing = [key for key in required if not tags.get(key)] + if missing: + raise SystemExit(f"Bootstrap stage is missing freeform tags: {', '.join(missing)}") + for key in required: + print(f"{key}={shlex.quote(tags[key])}") + PY + source "$${context_file}" + + case "$${cluster}" in + noprod) target_cluster_id="$${noprod_cluster_id}" ;; + prod) target_cluster_id="$${prod_cluster_id}" ;; + *) echo "Unsupported bootstrap cluster: $${cluster}" >&2; exit 1 ;; + esac + printf 'target_cluster_id=%q\ntarget_namespace=%q\n' \ + "$${target_cluster_id}" "$${namespace}" >>"$${context_file}" + + - type: Command + name: Validate namespace inputs + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/bootstrap-context.env" + + validate_k8s_dns_label() { + value="$1" + [ "$${#value}" -le 63 ] && printf '%s' "$${value}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + } + + printf '%s' "$${target_cluster_id}" | grep -Eq '^ocid1\.cluster\.' || { echo "Invalid OKE cluster OCID" >&2; exit 1; } + validate_k8s_dns_label "$${target_namespace}" || { echo "Invalid namespace: $${target_namespace}" >&2; exit 1; } + validate_k8s_dns_label "$${target_secret_name}" || { echo "Invalid secret name: $${target_secret_name}" >&2; exit 1; } + : "$${target_registry_username:?registry_username pipeline parameter is required}" + : "$${target_pull_password_secret_ocid:?pull_password_secret_ocid pipeline parameter is required}" + printf '%s' "$${target_pull_password_secret_ocid}" | grep -Eq '^ocid1\.vaultsecret\.' || { echo "Invalid pull password secret OCID" >&2; exit 1; } + + echo "Bootstrap inputs validated for $${application} on $${cluster}" + + - type: Command + name: Configure cluster access + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/bootstrap-context.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + oci ce cluster create-kubeconfig \ + --cluster-id "$${target_cluster_id}" \ + --file "$${KUBECONFIG}" \ + --region "$${OCI_CLI_REGION}" \ + --token-version 2.0.0 \ + --kube-endpoint "$${kube_endpoint}" + + - type: Command + name: Ensure namespace exists + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/bootstrap-context.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + kubectl create namespace "$${target_namespace}" --dry-run=client -o yaml | kubectl apply -f - + echo "Namespace initialized: $${target_namespace}" + + - type: Command + name: Create OCIR pull secret + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/bootstrap-context.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + ocir_pull_password="$(oci secrets secret-bundle get \ + --secret-id "$${target_pull_password_secret_ocid}" \ + --query 'data."secret-bundle-content".content' \ + --raw-output | base64 --decode)" + if [ -z "$${ocir_pull_password}" ]; then + echo "OCIR pull password secret is empty" >&2 + exit 1 + fi + + kubectl -n "$${target_namespace}" delete secret "$${target_secret_name}" --ignore-not-found=true + kubectl -n "$${target_namespace}" create secret docker-registry "$${target_secret_name}" \ + --docker-server="$${ocir_registry}" \ + --docker-username="$${target_registry_username}" \ + --docker-password="$${ocir_pull_password}" + echo "OCIR pull secret created: $${target_secret_name}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-Chart.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-Chart.yaml.tpl new file mode 100644 index 000000000..92a748d24 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-Chart.yaml.tpl @@ -0,0 +1,6 @@ +apiVersion: v2 +name: ${application_name} +description: Namespace umbrella chart for the ${application_name} application +type: application +version: ${chart_version} +appVersion: "1.0.0" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-README.md.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-README.md.tpl new file mode 100644 index 000000000..d27207960 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-README.md.tpl @@ -0,0 +1,17 @@ +# ${chart_repository_name} + +This repository contains Helm charts for the `${application_name}` application. + +The umbrella chart and component charts have separate lifecycles: + +- `${application_path}` is the `${application_name}` umbrella chart. It owns shared namespace baseline resources. +- `${application_path}/charts/${component_name}` is the `${component_name}` component chart. It owns the component `Deployment`, `Service`, and `ServiceAccount`. +- `${application_path}/.helmignore` excludes `charts/` when the baseline is packaged, so component charts are never installed by the umbrella release. + +Trigger behavior: + +- Changes under `${application_path}/charts/${component_name}/**` package only the component chart and redeploy dev. +- Changes under `${application_path}/**`, excluding `charts/**`, package only the umbrella baseline chart and start `${application_name}-deploy`. +- Source code changes are handled in the separate `${component_name}` source repository. + +The umbrella chart has no component dependencies. Component environments are deployed by installing each component chart directly with environment-specific values. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-helmignore.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-helmignore.tpl new file mode 100644 index 000000000..ee3892e87 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-helmignore.tpl @@ -0,0 +1 @@ +charts/ diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-values.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-values.yaml.tpl new file mode 100644 index 000000000..0090ae1c2 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-chart-values.yaml.tpl @@ -0,0 +1 @@ +# Shared namespace baseline values belong here. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-delivery-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-delivery-pipeline.yaml.tpl new file mode 100644 index 000000000..cfc512f2d --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-delivery-pipeline.yaml.tpl @@ -0,0 +1,84 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + application_chart_source_dir: "application-chart" + component_chart_repo_prefix: "${component_chart_repo_prefix}" + component_chart_path: "${component_chart_path}" + component_delivery_env_file: "/workspace/component-delivery.env" + component_name: "${component_name}" + component_image_repo_prefix: "${component_image_repo_prefix}" + repo_compartment_id: "${repo_compartment_id}" + region: "${region}" + region_key: "${region_key}" + tenancy_namespace: "${tenancy_namespace}" + exportedVariables: + - component_chart_version + - image_repository + - image_tag + +steps: + - type: Command + name: Select delivery actions + failImmediatelyOnError: true + command: | + export OCI_CLI_REGION="$${region}" + + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/application-delivery-select-actions.sh" \ + -a "$${application_chart_source_dir}" \ + -p "$${component_chart_path}" \ + -r "$${component_chart_repo_prefix}" \ + -o "$${component_delivery_env_file}" + + - type: Command + name: Build application image + failImmediatelyOnError: true + command: | + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/application-delivery-build-image.sh" \ + -e "$${component_delivery_env_file}" \ + -c "$${repo_compartment_id}" \ + -k "$${region_key}" \ + -p "$${component_image_repo_prefix}" \ + -t "$${tenancy_namespace}" + + - type: Command + name: Ensure component chart exists + failImmediatelyOnError: true + command: | + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/application-delivery-check-chart.sh" \ + -e "$${component_delivery_env_file}" \ + -k "$${region_key}" \ + -t "$${tenancy_namespace}" + + - type: Command + name: Package component chart + failImmediatelyOnError: true + command: | + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/application-delivery-package-chart.sh" \ + -e "$${component_delivery_env_file}" \ + -c "$${repo_compartment_id}" \ + -k "$${region_key}" \ + -t "$${tenancy_namespace}" + + - type: Command + name: Export component dev deployment parameters + failImmediatelyOnError: true + command: | + source "$${component_delivery_env_file}" + + if [ -z "$${chart_version:-}" ]; then + chart_version="$(bash "$${OCI_PRIMARY_SOURCE_DIR}/script/read-chart-version.sh" \ + -p "$${OCI_WORKSPACE_DIR}/$${chart_source_dir}/$${chart_path}")" + fi + component_chart_version="$${chart_version}" + + image_repository="${component_image_repository}" + image_tag="$(git -C "$${OCI_WORKSPACE_DIR}/$${component_name}" rev-parse --short=7 HEAD)" + export image_repository + export image_tag + export component_chart_version + + printf "Component dev deployment chart version: %s\n" "$${component_chart_version}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-README.md.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-README.md.tpl new file mode 100644 index 000000000..d2dadd1cb --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-README.md.tpl @@ -0,0 +1,37 @@ +# ${component_name} + +This repository contains the source code for the `${component_name}` component. + +Use this repository for application code, tests, and the component-owned pull request build spec. Do not put Helm charts or values here; chart changes belong in the separate `${chart_repository_name}` repository. + +Important files: + +- `Dockerfile`: image build definition used by `${component_name}-build`. +- `.oci-devops/application.env`: component identity read by the shared build pipeline. +- `.oci-devops/pull-request-pipeline.yaml`: component-owned PR validation build spec. + +Developer responsibilities: + +- The pull request pipeline spec is intentionally a placeholder. Testing strategies depend on the component language, framework, and integration boundaries, so replace `.oci-devops/pull-request-pipeline.yaml` with the checks that make sense for this component. +- The shared build pipeline expects the component build to be described by the `Dockerfile`. Prefer a multi-stage Dockerfile so compilation, dependency installation, and build tooling stay in builder stages while the final runtime image stays small and production-focused. + +Normal development flow: + +1. Open a pull request to `main`. +2. The `${component_name}-pr` pipeline runs the component-owned checks. +3. After merge, `${component_name}-build` builds a multi-architecture image tagged with the 7-character commit SHA. +4. The successful build deploys the dev release. +5. Run `${component_name}-release-build` with a release candidate tag such as `1.0.0-rc.1` to promote the selected image. +6. After production deployment and final Git tagging, OCI DevOps finishes by reporting the Helm release status, resources, history, notes, and namespace release listing. + +The build pipeline reads this contract file: + +```bash +component_name=${component_name} +``` + +Images are pushed to: + +```text +.ocir.io//${devops_project_prefix}/${application_name}/${component_name}: +``` diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-dockerfile.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-dockerfile.tpl new file mode 100644 index 000000000..020e8ceee --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-dockerfile.tpl @@ -0,0 +1,6 @@ +FROM container-registry.oracle.com/os/oraclelinux:10-slim + +LABEL org.opencontainers.image.title="${component_name}" +LABEL org.opencontainers.image.description="Sample application image for the OKE Helm starter stack" + +CMD ["/usr/bin/sleep", "31536000"] diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-env.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-env.tpl new file mode 100644 index 000000000..4de95fe37 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/application-source-env.tpl @@ -0,0 +1 @@ +component_name=${component_name} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-README.md.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-README.md.tpl new file mode 100644 index 000000000..84b9e5909 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-README.md.tpl @@ -0,0 +1,32 @@ +# Cluster Administration + +This repository owns cluster-wide Kubernetes objects and cluster-specific tool configuration for the OKE clusters connected to the `${project_name}` OCI DevOps project. + +## Layout + +- `catalog/tools.yaml`: stack-managed pinned public Helm charts mirrored into OCIR. +- `clusters//baseline`: non-namespaced Kubernetes objects applied with `kubectl apply --server-side`. +- `clusters//tools//values.yaml`: values for that tool on that physical cluster. +- `clusters//tools//tool.yaml`: tool name, namespace, and `depends_on` metadata generated when the target is added. +- `clusters//tools//resources`: supplemental namespaced objects applied after Helm. +- `clusters//tools//verify.sh`: optional post-deployment verification script. + +Changes must reach `main` through a pull request. The PR pipeline validates configuration and dependency cycles without touching a cluster. After merge, only changed stages are dispatched. Independent tools run in parallel dependency waves, supplemental resources follow their tool's Helm stage, and selected cluster-wide baseline resources run last so they can use CRDs installed by the charts. + +Before mirroring charts, requesting approval, or mutating a cluster, `cluster-admin-build` prints an execution plan containing the configuration commit, selected cluster, dependency waves, actions, namespaces, chart versions, values versions, and final baseline work. The production deployment includes a compact target summary in its display name and tags. + +Both cluster pipelines use the same shell orchestrator and immutable commit-versioned deployment plan. `cluster-admin-noprod` starts its orchestrator immediately; `cluster-admin-prod` requires approval first. Each orchestrator deploys only selected targets in dependency waves and applies baseline resources last. Cluster-specific values and supplemental resources remain under their respective `clusters/` directories. + +Cluster pipelines expose only `config_commit`, `cluster_id`, and `cluster_name`. The dispatcher supplies chart/value versions and tool identity as generic stage-level overrides, so adding tools does not expand the pipeline parameter list. + +Resource Manager regenerates only `catalog/tools.yaml` and adds missing cluster/tool directories when its cluster administration JSON changes. Other existing repository files are never replaced. Every JSON tool entry must include the public Helm repository, upstream chart name, and pinned version; edit those coordinates through Resource Manager rather than directly in the generated catalog. + +The repository field accepts either an HTTPS Helm repository or an OCI repository base path. OCI charts are resolved as `/`. Public OCI sources work without additional authentication; private sources require Helm credentials for the source registry. + +Removing a manifest does not delete the live object, and out-of-band drift is not continuously reconciled. Plain Kubernetes `Secret` resources are rejected; use External Secrets Operator resources instead. + +The orchestrators retain Helm releases unless the explicit decommission pipeline is run. Removing a file from `resources/` still does not prune the live object. + +Use `cluster-admin--decommission` before removing a tool from Git or Resource Manager. Supply the tool name and namespace, then type `DELETE /` exactly. The pipeline validates and deletes supplemental resources, uninstalls the Helm release, and retains the namespace for review. Production requires approval before removal. + +OCI resources created for this workflow carry freeform tags for `owner=cluster-administrators`, `purpose=cluster-administration`, and `scope=operations`, plus `cluster`, `tool`, and `role` where applicable. diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-build-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-build-pipeline.yaml.tpl new file mode 100644 index 000000000..0bb3acd5c --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-build-pipeline.yaml.tpl @@ -0,0 +1,53 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + artifact_repository_id: "${artifact_repository_id}" + compartment_id: "${compartment_id}" + mirror_pipeline_id: "${mirror_pipeline_id}" + project_id: "${project_id}" + region: "${region}" + region_key: "${region_key}" + tenancy_namespace: "${tenancy_namespace}" + chart_repo_prefix: "${chart_repo_prefix}" + target_file: "/workspace/cluster-admin-targets.json" + +steps: + - type: Command + name: Prepare validation dependencies + failImmediatelyOnError: true + command: | + set -euo pipefail + python3 -c 'import yaml' 2>/dev/null || python3 -m pip install --disable-pip-version-check PyYAML==6.0.2 + + - type: Command + name: Validate changed cluster configuration + failImmediatelyOnError: true + command: | + set -euo pipefail + commit="$${OCI_PRIMARY_SOURCE_COMMIT_HASH:-$${OCI_TRIGGER_COMMIT_HASH:-}}" + : "$${commit:?The cluster-admin source commit is unavailable}" + + python3 "$${OCI_PRIMARY_SOURCE_DIR}/script/validate-config.py" \ + --repo "$${OCI_PRIMARY_SOURCE_DIR}" \ + --commit "$${commit}" \ + --mode changed \ + --output "$${target_file}" + + - type: Command + name: Publish values and dispatch changed targets + failImmediatelyOnError: true + command: | + set -euo pipefail + export OCI_CLI_REGION="$${region}" + + python3 "$${OCI_PRIMARY_SOURCE_DIR}/script/publish-and-dispatch.py" \ + --repo "$${OCI_PRIMARY_SOURCE_DIR}" \ + --targets "$${target_file}" \ + --project-id "$${project_id}" \ + --mirror-pipeline-id "$${mirror_pipeline_id}" \ + --artifact-repository-id "$${artifact_repository_id}" \ + --compartment-id "$${compartment_id}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-decommission-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-decommission-command-spec.yaml.tpl new file mode 100644 index 000000000..ee8eb9354 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-decommission-command-spec.yaml.tpl @@ -0,0 +1,141 @@ +version: 0.1 +component: command +timeoutInSeconds: 1200 +shell: bash +env: + variables: + kube_endpoint: "${kube_endpoint}" + namespace: "$${tool_namespace}" + noprod_cluster_id: "${noprod_oke_cluster_id}" + prod_cluster_id: "${prod_oke_cluster_id}" + region: "${region}" + repository_id: "${repository_id}" + tool_name: "$${tool_name}" + +steps: + - type: Command + name: Validate decommission request + failImmediatelyOnError: true + command: | + set -euo pipefail + + validate_dns_label() { + value="$1" + [ "$${#value}" -le 63 ] && printf '%s' "$${value}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + } + + : "$${tool_name:?tool_name deployment parameter is required}" + : "$${namespace:?tool_namespace deployment parameter is required}" + : "$${OCI_STAGE_ID:?OCI_STAGE_ID is required to resolve the target cluster}" + + stage_json="$${OCI_WORKSPACE_DIR}/decommission-stage.json" + context_file="$${OCI_WORKSPACE_DIR}/decommission-context.env" + oci devops deploy-stage get --deploy-stage-id "$${OCI_STAGE_ID}" >"$${stage_json}" + python3 - "$${stage_json}" >"$${context_file}" <<'PY' + import json, shlex, sys + + stage = json.load(open(sys.argv[1], encoding="utf-8"))["data"] + tags = stage.get("freeform-tags") or stage.get("free-form-tags") or {} + cluster = tags.get("cluster") + if cluster not in ("noprod", "prod"): + raise SystemExit(f"Decommission stage has invalid cluster tag: {cluster!r}") + print(f"cluster_name={shlex.quote(cluster)}") + PY + source "$${context_file}" + case "$${cluster_name}" in + noprod) cluster_id="$${noprod_cluster_id}" ;; + prod) cluster_id="$${prod_cluster_id}" ;; + esac + printf 'cluster_id=%q\n' "$${cluster_id}" >>"$${context_file}" + + validate_dns_label "$${cluster_name}" || { echo "Invalid cluster_name: $${cluster_name}" >&2; exit 1; } + validate_dns_label "$${tool_name}" || { echo "Invalid tool_name: $${tool_name}" >&2; exit 1; } + validate_dns_label "$${namespace}" || { echo "Invalid tool_namespace: $${namespace}" >&2; exit 1; } + + echo "Decommission request validated for $${cluster_name}/$${tool_name} in namespace $${namespace}" + + - type: Command + name: Configure cluster access + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/decommission-context.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + oci ce cluster create-kubeconfig \ + --cluster-id "$${cluster_id}" \ + --file "$${KUBECONFIG}" \ + --region "$${region}" \ + --token-version 2.0.0 \ + --kube-endpoint "$${kube_endpoint}" + + - type: Command + name: Load current tool configuration + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/decommission-context.env" + archive="$${OCI_WORKSPACE_DIR}/cluster-admin.zip" + source_dir="$${OCI_WORKSPACE_DIR}/cluster-admin-source" + mkdir -p "$${source_dir}" + oci devops repository get-repository-archive-content \ + --repository-id "$${repository_id}" \ + --ref-name main \ + --format zip \ + --file "$${archive}" + unzip -q "$${archive}" -d "$${source_dir}" + + tool_dir="$(find "$${source_dir}" -type d -path "*/clusters/$${cluster_name}/tools/$${tool_name}" -print -quit)" + : "$${tool_dir:?Tool configuration must remain on main until decommission completes}" + printf 'tool_dir=%q\n' "$${tool_dir}" >"$${OCI_WORKSPACE_DIR}/decommission.env" + + - type: Command + name: Delete supplemental resources + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/decommission-context.env" + source "$${OCI_WORKSPACE_DIR}/decommission.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + resources_dir="$${tool_dir}/resources" + + if [ -d "$${resources_dir}" ] && find "$${resources_dir}" -maxdepth 1 -type f \( -name '*.yaml' -o -name '*.yml' \) | grep -q .; then + dry_run="$${OCI_WORKSPACE_DIR}/decommission-dry-run.json" + kubectl apply --server-side --dry-run=server --field-manager=oci-devops-cluster-admin \ + -n "$${namespace}" -f "$${resources_dir}" -o json >"$${dry_run}" + python3 - "$${dry_run}" "$${namespace}" <<'PY' + import json, sys + + text, expected = open(sys.argv[1], encoding="utf-8").read(), sys.argv[2] + decoder, index = json.JSONDecoder(), 0 + while index < len(text): + while index < len(text) and text[index].isspace(): + index += 1 + if index >= len(text): + break + obj, index = decoder.raw_decode(text, index) + for item in obj.get("items", [obj]): + namespace = item.get("metadata", {}).get("namespace") + if namespace != expected: + kind = item.get("kind") + name = item.get("metadata", {}).get("name") + raise SystemExit(f"Refusing to delete {kind}/{name} from namespace {namespace!r}; expected {expected!r}") + PY + kubectl delete -n "$${namespace}" -f "$${resources_dir}" --ignore-not-found=true + else + echo "No supplemental resources to delete for $${cluster_name}/$${tool_name}" + fi + + - type: Command + name: Uninstall tool chart + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/decommission-context.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + if helm status "$${tool_name}" -n "$${namespace}" >/dev/null 2>&1; then + helm uninstall "$${tool_name}" -n "$${namespace}" --wait --timeout 10m + echo "Uninstalled Helm release $${tool_name} from $${namespace}" + else + echo "Helm release $${tool_name} is already absent from $${namespace}" + fi + echo "Namespace $${namespace} was retained for explicit administrator review" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-deploy-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-deploy-command-spec.yaml.tpl new file mode 100644 index 000000000..8fa12a8ef --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-deploy-command-spec.yaml.tpl @@ -0,0 +1,100 @@ +version: 0.1 +component: command +timeoutInSeconds: 10000 +shell: bash +env: + variables: + artifact_repository_id: "${artifact_repository_id}" + chart_prefix: "${chart_prefix}" + cluster_id: "$${cluster_id}" + cluster_name: "$${cluster_name}" + config_commit: "$${config_commit}" + kube_endpoint: "${kube_endpoint}" + region: "${region}" + registry: "${registry}" + repository_id: "${repository_id}" + tenancy_namespace: "${tenancy_namespace}" + +steps: + - type: Command + name: Validate cluster deployment input + failImmediatelyOnError: true + command: | + set -euo pipefail + : "$${config_commit:?config_commit deployment parameter is required}" + : "$${cluster_id:?cluster_id deployment parameter is required}" + case "$${cluster_name}" in noprod|prod) ;; *) echo "cluster_name must be noprod or prod" >&2; exit 1;; esac + printf '%s' "$${config_commit}" | grep -Eq '^[0-9a-f]{40}$' || { echo "config_commit must be a full Git SHA" >&2; exit 1; } + + - type: Command + name: Configure cluster access + failImmediatelyOnError: true + command: | + set -euo pipefail + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + oci ce cluster create-kubeconfig \ + --cluster-id "$${cluster_id}" \ + --file "$${KUBECONFIG}" \ + --region "$${region}" \ + --token-version 2.0.0 \ + --kube-endpoint "$${kube_endpoint}" + + - type: Command + name: Download exact validated configuration + failImmediatelyOnError: true + command: | + set -euo pipefail + archive="$${OCI_WORKSPACE_DIR}/cluster-admin.zip" + source_root="$${OCI_WORKSPACE_DIR}/cluster-admin-source" + targets="$${OCI_WORKSPACE_DIR}/cluster-admin-targets.json" + mkdir -p "$${source_root}" + oci artifacts generic artifact download-by-path \ + --repository-id "$${artifact_repository_id}" \ + --artifact-path cluster-admin/deployment-plan.json \ + --artifact-version "$${config_commit}" \ + --file "$${targets}" + oci devops repository get-repository-archive-content \ + --repository-id "$${repository_id}" \ + --ref-name "$${config_commit}" \ + --format zip \ + --file "$${archive}" + unzip -q "$${archive}" -d "$${source_root}" + catalog_file="$(find "$${source_root}" -type f -path '*/catalog/tools.yaml' -print -quit)" + : "$${catalog_file:?catalog/tools.yaml not found in repository archive}" + repo_dir="$(dirname "$(dirname "$${catalog_file}")")" + printf 'repo_dir=%q\n' "$${repo_dir}" >"$${OCI_WORKSPACE_DIR}/cluster-deploy.env" + + - type: Command + name: Authenticate to mirrored chart registry + failImmediatelyOnError: true + command: | + set -euo pipefail + token_file="$${OCI_WORKSPACE_DIR}/ocir-token.json" + oci raw-request \ + --http-method GET \ + --target-uri "https://$${registry}/20180419/docker/token" >"$${token_file}" + python3 - "$${token_file}" <<'PY' | helm registry login "$${registry}" -u BEARER_TOKEN --password-stdin + import json, sys + payload = json.load(open(sys.argv[1], encoding="utf-8")) + data = payload.get("data", payload) + if isinstance(data, str): + data = json.loads(data) + print(data["token"]) + PY + + - type: Command + name: Deploy selected cluster changes + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/cluster-deploy.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + python3 "$${repo_dir}/script/deploy-cluster.py" \ + --repo "$${repo_dir}" \ + --targets "$${OCI_WORKSPACE_DIR}/cluster-admin-targets.json" \ + --cluster "$${cluster_name}" \ + --commit "$${config_commit}" \ + --artifact-repository-id "$${artifact_repository_id}" \ + --registry "$${registry}" \ + --tenancy-namespace "$${tenancy_namespace}" \ + --chart-prefix "$${chart_prefix}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-mirror-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-mirror-pipeline.yaml.tpl new file mode 100644 index 000000000..6b21df247 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-mirror-pipeline.yaml.tpl @@ -0,0 +1,34 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + compartment_id: "${compartment_id}" + region: "${region}" + registry: "${region_key}.ocir.io" + tenancy_namespace: "${tenancy_namespace}" + chart_repo_prefix: "${chart_repo_prefix}" + +steps: + - type: Command + name: Prepare mirror dependencies + failImmediatelyOnError: true + command: | + set -euo pipefail + python3 -c 'import yaml' 2>/dev/null || python3 -m pip install --disable-pip-version-check PyYAML==6.0.2 + + - type: Command + name: Mirror missing tool charts + failImmediatelyOnError: true + command: | + set -euo pipefail + export OCI_CLI_REGION="$${region}" + + python3 "$${OCI_PRIMARY_SOURCE_DIR}/script/mirror-charts.py" \ + --catalog "$${OCI_PRIMARY_SOURCE_DIR}/catalog/tools.yaml" \ + --compartment-id "$${compartment_id}" \ + --registry "$${registry}" \ + --tenancy-namespace "$${tenancy_namespace}" \ + --target-prefix "$${chart_repo_prefix}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-pr-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-pr-pipeline.yaml.tpl new file mode 100644 index 000000000..ec943d9ed --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-pr-pipeline.yaml.tpl @@ -0,0 +1,22 @@ +version: 0.1 +component: build +timeoutInSeconds: 3000 +runAs: root +shell: bash + +steps: + - type: Command + name: Validate cluster administration configuration + failImmediatelyOnError: true + command: | + set -euo pipefail + python3 -c 'import yaml' 2>/dev/null || python3 -m pip install --disable-pip-version-check PyYAML==6.0.2 + + commit="$${OCI_PRIMARY_SOURCE_COMMIT_HASH:-HEAD}" + python3 "$${OCI_PRIMARY_SOURCE_DIR}/script/validate-config.py" \ + --repo "$${OCI_PRIMARY_SOURCE_DIR}" \ + --commit "$${commit}" \ + --mode all \ + --output "/workspace/cluster-admin-validation.json" + + echo "Cluster administration validation succeeded" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-tools.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-tools.yaml.tpl new file mode 100644 index 000000000..1a9c489ea --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/cluster-admin-tools.yaml.tpl @@ -0,0 +1,7 @@ +tools: +%{ for tool in tools ~} + ${tool.name}: + repository: ${jsonencode(tool.repository)} + chart: ${jsonencode(tool.chart)} + version: ${jsonencode(tool.version)} +%{ endfor ~} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-Chart.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-Chart.yaml.tpl new file mode 100644 index 000000000..98350a44e --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-Chart.yaml.tpl @@ -0,0 +1,6 @@ +apiVersion: v2 +name: ${component_name} +description: Sample application component workload subchart for the OKE Helm starter stack +type: application +version: ${chart_version} +appVersion: "1.0.0" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-deployment.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-deployment.yaml.tpl new file mode 100644 index 000000000..685af48fd --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-deployment.yaml.tpl @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.nameOverride }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + spec: + serviceAccountName: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.serviceAccount.name }} + automountServiceAccountToken: false + containers: + - name: app + {{- $imageTag := .Values.image.tag }} + {{- if eq .Values.environment "prod" }} + {{- $imageTag = regexReplaceAll "-rc\\.[0-9]+$" .Values.image.tag "" }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ $imageTag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/bin/sleep", "31536000"] + ports: + - name: http + containerPort: 8080 + resources: + requests: + cpu: {{ .Values.resources.requests.cpu | quote }} + memory: {{ .Values.resources.requests.memory | quote }} + limits: + memory: {{ .Values.resources.limits.memory | quote }} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-service.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-service.yaml.tpl new file mode 100644 index 000000000..520280ed0 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-service.yaml.tpl @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.nameOverride }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +spec: + selector: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-serviceaccount.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-serviceaccount.yaml.tpl new file mode 100644 index 000000000..2cf64e696 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-serviceaccount.yaml.tpl @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ default (ternary (printf "%s-%s" .Values.component.name .Values.environment) .Values.component.name (and (ne .Values.environment "") (ne .Values.environment "prod"))) .Values.serviceAccount.name }} + labels: + app.kubernetes.io/name: {{ .Values.application.name }} + app.kubernetes.io/component: {{ .Values.component.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.environment }} + env: {{ .Values.environment }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountToken }} +{{- with .Values.serviceAccount.imagePullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ .name }} +{{- end }} +{{- end }} diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-values.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-values.yaml.tpl new file mode 100644 index 000000000..896bf0b69 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-chart-values.yaml.tpl @@ -0,0 +1,30 @@ +replicaCount: 1 +environment: "" +nameOverride: "" + +application: + name: ${application_name} + +component: + name: ${component_name} + +image: + repository: ghcr.io/oracle/oraclelinux + tag: "9" + pullPolicy: IfNotPresent + +serviceAccount: + name: "" + automountToken: false + imagePullSecrets: + - name: ocirsecret + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 128Mi + +service: + port: 80 diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-values.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-values.yaml.tpl new file mode 100644 index 000000000..c1e6b31ed --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/component-values.yaml.tpl @@ -0,0 +1,8 @@ +environment: "${environment}" +application: + name: "${application_chart_name}" +component: + name: "${component_name}" +image: + repository: "$${image_repository}" + tag: "$${${image_tag_parameter}}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/helm-chart-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/helm-chart-pipeline.yaml.tpl new file mode 100644 index 000000000..5e9c47fb1 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/helm-chart-pipeline.yaml.tpl @@ -0,0 +1,41 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + CHART_VALUES_FILE: "" + repo_compartment_id: "${repo_compartment_id}" + region: "${region}" + region_key: "${region_key}" + tenancy_namespace: "${tenancy_namespace}" + +steps: + - type: Command + name: Package and push Helm chart + failImmediatelyOnError: true + command: | + export OCI_CLI_REGION="$${region}" + + : "$${chart_name:?chart_name pipeline parameter is required}" + : "$${chart_version:?chart_version pipeline parameter is required}" + : "$${chart_path:?chart_path pipeline parameter is required}" + : "$${chart_source_dir:?chart_source_dir pipeline parameter is required}" + : "$${chart_repo_prefix:?chart_repo_prefix pipeline parameter is required}" + + cd "$${OCI_WORKSPACE_DIR}/$${chart_source_dir}" + args=( + -c "$${repo_compartment_id}" + -k "$${region_key}" + -n "$${chart_name}" + -v "$${chart_version}" + -d "$${chart_path}" + -p "$${chart_repo_prefix}" + -t "$${tenancy_namespace}" + ) + if [ -n "$${CHART_VALUES_FILE:-}" ]; then + args+=(-f "$${CHART_VALUES_FILE}") + fi + "$${OCI_PRIMARY_SOURCE_DIR}/script/package-push-chart.sh" \ + "$${args[@]}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/pipelines-README.md.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/pipelines-README.md.tpl new file mode 100644 index 000000000..a2502d342 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/pipelines-README.md.tpl @@ -0,0 +1,25 @@ +# Pipelines + +This repository contains reusable OCI DevOps build specs and helper scripts for the ${application_name} delivery flow. + +It is the primary source for shared build pipelines. The component source repository and chart repository are checked out as secondary sources when a pipeline needs them. + +Generated build specs: + +- `${component_name}-build-pipeline.yaml` builds component source changes, packages that component chart when needed, and exports deployment parameters for dev. +- `${application_name}-package-pipeline.yaml` packages the `${application_name}` umbrella baseline chart and starts the baseline deployment pipeline. +- `${component_name}-release-pipeline.yaml` creates the OCI DevOps Git tag, retags the matching 7-character SHA image as a SemVer release candidate such as `1.0.0-rc.1`, and starts the component release deployment. +- `helm-chart-pipeline.yaml` is a generic chart packager retained for future reuse. + +Resource Manager seeds these files as starter content. Existing files are developer-owned and are never overwritten by later stack applies. + +Components may instead configure `build_spec_path` in the Resource Manager applications JSON. An explicit path can be shared, for example `java/java-build-pipeline.yaml`. Resource Manager creates the file and parent folders from the default template only when missing, giving the DevOps engineer a working base. The path becomes user-owned after that first commit and is never refreshed or overwritten. + +The build specs should stay readable and macro-oriented. Detailed branching, registry checks, chart parsing, image promotion, and release input resolution belong in `script/*.sh`. + +Default pipeline names: + +- `${component_name}-build` +- `${application_name}-package` +- `${component_name}-pr` +- `${component_name}-release-build` diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/promote-release-image-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/promote-release-image-command-spec.yaml.tpl new file mode 100644 index 000000000..a8e5e9ad4 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/promote-release-image-command-spec.yaml.tpl @@ -0,0 +1,108 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +env: + variables: + OCI_CLI_REGION: "${region}" + image_repository: "$${image_repository}" + release_candidate_tag: "$${image_tag}" + exportedVariables: + - release_image_tag + +steps: + - type: Command + name: Validate release image promotion inputs + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + + : "$${image_repository:?image_repository deployment parameter is required}" + : "$${release_candidate_tag:?image_tag deployment parameter is required}" + + if ! printf '%s' "$${image_repository}" | grep -Eq '^[a-z0-9.-]+[.]ocir[.]io/.+/.+'; then + echo "Invalid OCIR image repository: $${image_repository}" >&2 + exit 1 + fi + if ! printf '%s' "$${release_candidate_tag}" | grep -Eq '^[0-9]+[.][0-9]+[.][0-9]+-rc[.][0-9]+$'; then + echo "Invalid release candidate tag: $${release_candidate_tag}" >&2 + exit 1 + fi + + release_image_tag="$${release_candidate_tag%-rc.*}" + if ! printf '%s' "$${release_image_tag}" | grep -Eq '^[0-9]+[.][0-9]+[.][0-9]+$'; then + echo "Invalid final release tag: $${release_image_tag}" >&2 + exit 1 + fi + export release_image_tag + + { + printf "image_repository=%q\n" "$${image_repository}" + printf "release_candidate_tag=%q\n" "$${release_candidate_tag}" + printf "release_image_tag=%q\n" "$${release_image_tag}" + } > "$${OCI_WORKSPACE_DIR}/release-image-promotion.env" + + echo "Release image promotion inputs validated" + + - type: Command + name: Promote release image tag + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/release-image-promotion.env" + export release_image_tag + + registry="$${image_repository%%/*}" + repository_path="$${image_repository#*/}" + manifest_url="https://$${registry}/v2/$${repository_path}/manifests" + accept_header='application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' + + token="$(oci raw-request --http-method GET --target-uri "https://$${registry}/20180419/docker/token" | tr -d '\n' | sed -E 's/.*"token"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')" + if [ -z "$${token}" ]; then + echo "Failed to fetch OCIR bearer token" >&2 + exit 1 + fi + + manifest_file="$(mktemp)" + headers_file="$(mktemp)" + if curl -fsS -D "$${headers_file}" \ + -H "Authorization: Bearer $${token}" \ + -H "Accept: $${accept_header}" \ + "$${manifest_url}/$${release_image_tag}" >/dev/null; then + echo "Final release image tag already exists: $${image_repository}:$${release_image_tag}" >&2 + exit 1 + fi + + curl -fsS -D "$${headers_file}" \ + -H "Authorization: Bearer $${token}" \ + -H "Accept: $${accept_header}" \ + "$${manifest_url}/$${release_candidate_tag}" \ + -o "$${manifest_file}" + + source_digest="$(awk 'BEGIN{IGNORECASE=1} /^Docker-Content-Digest:/ {gsub("\r", "", $2); print $2}' "$${headers_file}" | tail -1)" + content_type="$(awk 'BEGIN{IGNORECASE=1} /^Content-Type:/ {$1=""; sub(/^ /, ""); gsub("\r", ""); print}' "$${headers_file}" | tail -1)" + if [ -z "$${source_digest}" ] || [ -z "$${content_type}" ]; then + echo "Unable to read source image manifest metadata" >&2 + exit 1 + fi + + curl -fsS -X PUT \ + -H "Authorization: Bearer $${token}" \ + -H "Content-Type: $${content_type}" \ + --data-binary "@$${manifest_file}" \ + "$${manifest_url}/$${release_image_tag}" >/dev/null + + curl -fsS -D "$${headers_file}" \ + -H "Authorization: Bearer $${token}" \ + -H "Accept: $${accept_header}" \ + "$${manifest_url}/$${release_image_tag}" >/dev/null + target_digest="$(awk 'BEGIN{IGNORECASE=1} /^Docker-Content-Digest:/ {gsub("\r", "", $2); print $2}' "$${headers_file}" | tail -1)" + if [ "$${source_digest}" != "$${target_digest}" ]; then + echo "Final release image digest mismatch: $${source_digest} != $${target_digest}" >&2 + exit 1 + fi + + echo "Promoted $${image_repository}:$${release_candidate_tag} to $${image_repository}:$${release_image_tag}" + echo "Digest: $${target_digest}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/release-pipeline.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/release-pipeline.yaml.tpl new file mode 100644 index 000000000..7c6510db6 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/release-pipeline.yaml.tpl @@ -0,0 +1,84 @@ +version: 0.1 +component: build +timeoutInSeconds: 10000 +runAs: root +shell: bash +env: + variables: + application_chart_source_dir: "application-chart" + application_chart_path: "${application_chart_path}" + component_chart_repo_prefix: "${component_chart_repo_prefix}" + application_release_env_file: "/workspace/application-release.env" + application_release_image_env_file: "/workspace/application-release-image.env" + component_image_repository: "${component_image_repository}" + component_image_repo_prefix: "${component_image_repo_prefix}" + repo_compartment_id: "${repo_compartment_id}" + region: "${region}" + region_key: "${region_key}" + tenancy_namespace: "${tenancy_namespace}" + exportedVariables: + - component_chart_version + - image_repository + - image_tag + +steps: + - type: Command + name: Resolve release inputs + failImmediatelyOnError: true + command: | + export OCI_CLI_REGION="$${region}" + + : "$${release_tag:?release_tag pipeline parameter is required}" + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/release-resolve-inputs.sh" \ + -g "$${release_tag}" \ + -i "$${commit_id:-}" \ + -a "$${application_chart_source_dir}" \ + -p "$${application_chart_path}" \ + -r "$${component_chart_repo_prefix}" \ + -o "$${application_release_env_file}" + + - type: Command + name: Promote application image + failImmediatelyOnError: true + command: | + source "$${application_release_env_file}" + + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/release-image-tag.sh" \ + -c "$${repo_compartment_id}" \ + -k "$${region_key}" \ + -n "$${component_name}" \ + -p "$${component_image_repo_prefix}" \ + -t "$${tenancy_namespace}" \ + -g "$${release_tag}" \ + -i "$${resolved_commit_id}" \ + -r "$${source_repository_id}" \ + -o "$${application_release_image_env_file}" + + - type: Command + name: Verify application chart + failImmediatelyOnError: true + command: | + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/release-verify-chart.sh" \ + -e "$${application_release_env_file}" \ + -k "$${region_key}" \ + -t "$${tenancy_namespace}" + + - type: Command + name: Print release summary + failImmediatelyOnError: true + command: | + source "$${application_release_env_file}" + source "$${application_release_image_env_file}" + + component_chart_version="$${chart_version}" + image_repository="$${component_image_repository}" + image_tag="$${promoted_image_tag}" + export component_chart_version + export image_repository + export image_tag + + bash "$${OCI_PRIMARY_SOURCE_DIR}/script/release-print-summary.sh" \ + -e "$${application_release_env_file}" \ + -p "$${application_release_image_env_file}" + + printf "Component staging deployment chart version: %s\n" "$${component_chart_version}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/tag-release-commit-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/tag-release-commit-command-spec.yaml.tpl new file mode 100644 index 000000000..d375a03ff --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/tag-release-commit-command-spec.yaml.tpl @@ -0,0 +1,68 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +env: + variables: + OCI_CLI_REGION: "${region}" + source_repository_id: "$${source_repository_id}" + release_candidate_tag: "$${image_tag}" + +steps: + - type: Command + name: Tag released commit + timeoutInSeconds: 600 + failImmediatelyOnError: true + command: | + set -euo pipefail + + : "$${source_repository_id:?source repository id is required}" + : "$${release_candidate_tag:?image_tag deployment parameter is required}" + + if ! printf '%s' "$${source_repository_id}" | grep -Eq '^ocid1[.]devopsrepository[.]'; then + echo "Invalid source repository OCID: $${source_repository_id}" >&2 + exit 1 + fi + if ! printf '%s' "$${release_candidate_tag}" | grep -Eq '^[0-9]+[.][0-9]+[.][0-9]+-rc[.][0-9]+$'; then + echo "Invalid release candidate tag: $${release_candidate_tag}" >&2 + exit 1 + fi + + release_tag="$${release_candidate_tag%-rc.*}" + if ! printf '%s' "$${release_tag}" | grep -Eq '^[0-9]+[.][0-9]+[.][0-9]+$'; then + echo "Invalid final release tag: $${release_tag}" >&2 + exit 1 + fi + + rc_commit_id="$(oci devops repository list-refs \ + --repository-id "$${source_repository_id}" \ + --ref-type TAG \ + --ref-name "$${release_candidate_tag}" \ + --query 'data.items[0]."object-id"' \ + --raw-output)" + if [ -z "$${rc_commit_id}" ] || [ "$${rc_commit_id}" = "null" ]; then + echo "Release candidate Git tag not found: $${release_candidate_tag}" >&2 + exit 1 + fi + + final_commit_id="$(oci devops repository list-refs \ + --repository-id "$${source_repository_id}" \ + --ref-type TAG \ + --ref-name "$${release_tag}" \ + --query 'data.items[0]."object-id"' \ + --raw-output)" + if [ -n "$${final_commit_id}" ] && [ "$${final_commit_id}" != "null" ]; then + if [ "$${final_commit_id}" = "$${rc_commit_id}" ]; then + echo "Final release Git tag already points to the released commit: $${release_tag}" + exit 0 + fi + echo "Final release Git tag $${release_tag} already points to a different commit: $${final_commit_id}" >&2 + exit 1 + fi + + oci devops repository create-or-update-git-tag-details \ + --repository-id "$${source_repository_id}" \ + --ref-name "$${release_tag}" \ + --object-id "$${rc_commit_id}" >/dev/null + + echo "Tagged released commit $${rc_commit_id} as $${release_tag}" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/verify-component-production-command-spec.yaml.tpl b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/verify-component-production-command-spec.yaml.tpl new file mode 100644 index 000000000..62ee97e8b --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/templates/verify-component-production-command-spec.yaml.tpl @@ -0,0 +1,67 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +env: + variables: + kube_endpoint: "${kube_endpoint}" + OCI_CLI_REGION: "${region}" + prod_cluster_id: "${prod_oke_cluster_id}" + +steps: + - type: Command + name: Resolve production release + failImmediatelyOnError: true + command: | + set -euo pipefail + : "$${OCI_STAGE_ID:?OCI_STAGE_ID is required to resolve the production release}" + + stage_json="$${OCI_WORKSPACE_DIR}/production-verification-stage.json" + context_file="$${OCI_WORKSPACE_DIR}/production-verification.env" + oci devops deploy-stage get --stage-id "$${OCI_STAGE_ID}" >"$${stage_json}" + python3 - "$${stage_json}" >"$${context_file}" <<'PY' + import json, shlex, sys + + stage = json.load(open(sys.argv[1], encoding="utf-8"))["data"] + tags = stage.get("freeform-tags") or stage.get("free-form-tags") or {} + required = ("namespace", "release") + missing = [key for key in required if not tags.get(key)] + if missing: + raise SystemExit(f"Production verification stage is missing freeform tags: {', '.join(missing)}") + for key in required: + print(f"{key}={shlex.quote(tags[key])}") + PY + + - type: Command + name: Configure production cluster access + failImmediatelyOnError: true + command: | + set -euo pipefail + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + oci ce cluster create-kubeconfig \ + --cluster-id "$${prod_cluster_id}" \ + --file "$${KUBECONFIG}" \ + --region "$${OCI_CLI_REGION}" \ + --token-version 2.0.0 \ + --kube-endpoint "$${kube_endpoint}" + + - type: Command + name: Report production Helm release + failImmediatelyOnError: true + command: | + set -euo pipefail + source "$${OCI_WORKSPACE_DIR}/production-verification.env" + export KUBECONFIG="$${OCI_WORKSPACE_DIR}/kubeconfig" + command -v helm >/dev/null 2>&1 || { echo "helm is not available in the deployment shell runner" >&2; exit 1; } + + echo "=== Release status and resources ===" + helm status "$${release}" --namespace "$${namespace}" --show-resources + + echo "=== Release history ===" + helm history "$${release}" --namespace "$${namespace}" --max 10 + + echo "=== Release notes ===" + helm get notes "$${release}" --namespace "$${namespace}" + + echo "=== Namespace release listing ===" + helm list --namespace "$${namespace}" --filter "^$${release}$" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_cluster_admin.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_cluster_admin.py new file mode 100644 index 000000000..6a0b089c5 --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_cluster_admin.py @@ -0,0 +1,495 @@ +import importlib.util +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name, filename): + spec = importlib.util.spec_from_file_location(name, ROOT / "repos/cluster-admin/script" / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +validate_config = load_script("validate_config", "validate-config.py") +dispatch = load_script("publish_and_dispatch", "publish-and-dispatch.py") +mirror = load_script("mirror_charts", "mirror-charts.py") +cluster_deploy = load_script("deploy_cluster", "deploy-cluster.py") + + +class ClusterAdminValidationTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.repo = Path(self.temp.name) + self.write("catalog/tools.yaml", """ +tools: + cert-manager: + repository: https://charts.example.test + chart: cert-manager + version: 1.0.0 + traefik: + repository: https://charts.example.test + chart: traefik-controller + version: 2.0.0 +""") + for cluster in ("noprod", "prod"): + self.write(f"clusters/{cluster}/baseline/README.md", "baseline\n") + self.write("clusters/noprod/tools/cert-manager/tool.yaml", """ +name: cert-manager +namespace: cert-manager +depends_on: [] +""") + self.write("clusters/noprod/tools/cert-manager/values.yaml", "replicas: 1\n") + self.write("clusters/noprod/tools/traefik/tool.yaml", """ +name: traefik +namespace: traefik +depends_on: + - cert-manager +""") + self.write("clusters/noprod/tools/traefik/values.yaml", "replicas: 1\n") + self.git("init", "-b", "main") + self.git("config", "user.email", "test@example.com") + self.git("config", "user.name", "Test") + self.commit("Initial configuration") + + def tearDown(self): + self.temp.cleanup() + + def write(self, relative, content): + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content.strip() + "\n", encoding="utf-8") + + def git(self, *arguments): + return subprocess.check_output(["git", "-C", str(self.repo), *arguments], text=True).strip() + + def commit(self, message): + self.git("add", ".") + self.git("commit", "-m", message) + return self.git("rev-parse", "HEAD") + + def targets(self, commit): + result = validate_config.validate(self.repo, commit, "changed") + return {(target["kind"], target.get("tool")): target for target in result["targets"]} + + def test_values_change_selects_helm_only(self): + self.write("clusters/noprod/tools/traefik/values.yaml", "replicas: 2\n") + targets = self.targets(self.commit("Change Traefik values")) + self.assertEqual(targets[("tool", "traefik")]["actions"], ["helm"]) + self.assertEqual(len(targets), 1) + + def test_prerequisite_change_selects_downstream_tool(self): + self.write("clusters/noprod/tools/cert-manager/values.yaml", "replicas: 2\n") + targets = self.targets(self.commit("Change cert-manager values")) + self.assertEqual(targets[("tool", "cert-manager")]["actions"], ["helm"]) + self.assertEqual(targets[("tool", "traefik")]["actions"], ["helm", "resources"]) + + def test_resource_change_selects_resources_only(self): + self.write( + "clusters/noprod/tools/traefik/resources/configmap.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: test\n", + ) + targets = self.targets(self.commit("Add Traefik resource")) + self.assertEqual(targets[("tool", "traefik")]["actions"], ["resources"]) + + def test_dependency_cycle_is_rejected(self): + self.write("clusters/noprod/tools/cert-manager/tool.yaml", """ +name: cert-manager +namespace: cert-manager +depends_on: + - traefik +""") + commit = self.commit("Create dependency cycle") + with self.assertRaisesRegex(ValueError, "dependency cycle"): + validate_config.validate(self.repo, commit, "changed") + + def test_logical_tool_name_can_differ_from_upstream_chart(self): + result = validate_config.validate(self.repo, self.git("rev-parse", "HEAD"), "all") + traefik = next( + target for target in result["targets"] + if target.get("tool") == "traefik" + ) + self.assertEqual(traefik["chart_version"], "2.0.0") + self.assertEqual(traefik["chart"], "traefik-controller") + + def test_oci_chart_repository_is_valid(self): + self.write("catalog/tools.yaml", """ +tools: + cert-manager: + repository: https://charts.example.test + chart: cert-manager + version: 1.0.0 + traefik: + repository: https://charts.example.test + chart: traefik-controller + version: 2.0.0 + external-dns: + repository: oci://registry-1.docker.io/bitnamicharts + chart: external-dns + version: 9.0.3 +""") + self.write("clusters/noprod/tools/external-dns/tool.yaml", """ +name: external-dns +namespace: external-dns +depends_on: [] +""") + self.write("clusters/noprod/tools/external-dns/values.yaml", "provider: oci\n") + commit = self.commit("Add OCI chart source") + + result = validate_config.validate(self.repo, commit, "all") + external_dns = next( + target for target in result["targets"] + if target.get("tool") == "external-dns" + ) + self.assertEqual(external_dns["chart_version"], "9.0.3") + + +class DependencyWaveTests(unittest.TestCase): + def test_cluster_admin_feature_is_opt_in_and_gates_resource_maps(self): + root_variables = (ROOT / "variables.tf").read_text(encoding="utf-8") + module_locals = (ROOT / "modules/devops/locals.tf").read_text(encoding="utf-8") + schema = (ROOT / "schema.yaml").read_text(encoding="utf-8") + + self.assertRegex( + root_variables, + r'variable "enable_cluster_admin" \{[^}]*default\s*=\s*false', + ) + self.assertIn( + 'cluster_admin_singleton = var.enable_cluster_admin ? { enabled = true } : {}', + module_locals, + ) + self.assertIn('cluster_admin_clusters = var.enable_cluster_admin ? {', module_locals) + self.assertIn('visible: ${enable_cluster_admin}', schema) + self.assertRegex(schema, r'development_mode:\n(?:.*\n){0,6}\s+visible: false') + self.assertIn('"name": "keda"', schema) + self.assertIn('"name": "kube-prometheus"', schema) + self.assertIn('"repository": "https://kedacore.github.io/charts"', schema) + self.assertIn('"chart": "kube-prometheus-stack"', schema) + self.assertIn('"version": "87.10.1"', schema) + self.assertIn('"name": "sample-api"', schema) + self.assertIn('"name": "sample-worker"', schema) + + def test_cluster_admin_schema_default_is_pretty_printed_json(self): + schema = yaml.safe_load((ROOT / "schema.yaml").read_text(encoding="utf-8")) + default = schema["variables"]["cluster_administration"]["default"] + + self.assertGreater(len(default.splitlines()), 10) + self.assertIn('\n "repository": "https://kedacore.github.io/charts",', default) + self.assertEqual(json.loads(default)["tools"][1]["chart"], "kube-prometheus-stack") + + def test_cluster_admin_artifact_repository_name_is_configurable(self): + root_variables = (ROOT / "variables.tf").read_text(encoding="utf-8") + root_main = (ROOT / "main.tf").read_text(encoding="utf-8") + module_repositories = ( + ROOT / "modules/devops/cluster_admin_repositories.tf" + ).read_text(encoding="utf-8") + schema = yaml.safe_load((ROOT / "schema.yaml").read_text(encoding="utf-8")) + + self.assertIn('variable "cluster_admin_artifact_repository_name"', root_variables) + self.assertIn( + "cluster_admin_artifact_repository_name = var.cluster_admin_artifact_repository_name", + root_main, + ) + self.assertIn('"${local.project_repo_prefix}-cluster-admin-values"', module_repositories) + field = schema["variables"]["cluster_admin_artifact_repository_name"] + self.assertEqual(field["default"], "") + self.assertEqual(field["visible"], "${enable_cluster_admin}") + self.assertEqual(field["maxLength"], 255) + self.assertRegex("operations-artifacts_1", field["pattern"]) + self.assertRegex("", field["pattern"]) + self.assertNotRegex(" invalid", field["pattern"]) + self.assertNotRegex("invalid/child", field["pattern"]) + + def test_cluster_admin_is_the_last_resource_manager_section(self): + schema = yaml.safe_load((ROOT / "schema.yaml").read_text(encoding="utf-8")) + self.assertEqual( + schema["variableGroups"][-1]["title"], + "Cluster Administration (Optional)", + ) + + def test_resource_manager_surfaces_terraform_input_constraints(self): + schema = yaml.safe_load((ROOT / "schema.yaml").read_text(encoding="utf-8")) + fields = schema["variables"] + + project = fields["devops_project_name"] + self.assertEqual(project["minLength"], 1) + self.assertEqual(project["maxLength"], 100) + self.assertRegex("oke-devops-starter", project["pattern"]) + self.assertNotRegex(" ", project["pattern"]) + + secret = fields["namespace_init_secret_name"] + self.assertEqual(secret["minLength"], 1) + self.assertEqual(secret["maxLength"], 63) + self.assertRegex("ocirsecret", secret["pattern"]) + self.assertNotRegex("OCISecret", secret["pattern"]) + + applications = fields["applications"] + self.assertRegex('[\n {"name": "app", "components": []}\n]', applications["pattern"]) + self.assertNotRegex('{"tools": []}', applications["pattern"]) + self.assertEqual(applications["maxLength"], 8192) + + cluster_admin = fields["cluster_administration"] + self.assertRegex('{\n "tools": []\n}', cluster_admin["pattern"]) + self.assertNotRegex('[]', cluster_admin["pattern"]) + self.assertEqual(cluster_admin["maxLength"], 8192) + + def test_cluster_wide_resources_are_dispatched_after_tools(self): + source = (ROOT / "repos/cluster-admin/script/deploy-cluster.py").read_text( + encoding="utf-8" + ) + tool_dispatch = source.index("for number, wave in enumerate") + baseline_dispatch = source.index( + 'if any(target["kind"] == "baseline" for target in targets)' + ) + self.assertLess(tool_dispatch, baseline_dispatch) + + def test_cluster_deploy_supports_managed_runner_python(self): + source = (ROOT / "repos/cluster-admin/script/deploy-cluster.py").read_text( + encoding="utf-8" + ) + self.assertNotIn("text=True", source) + self.assertIn("universal_newlines=True", source) + + def test_both_clusters_use_the_shared_orchestrator(self): + source = (ROOT / "modules/devops/cluster_admin_deploy_pipelines.tf").read_text( + encoding="utf-8" + ) + self.assertIn('resource "oci_devops_deploy_stage" "cluster_admin_orchestrator"', source) + self.assertIn("for_each = local.cluster_admin_clusters", source) + self.assertNotIn("cluster_admin_dag_tool_helm", source) + self.assertNotIn("cluster_admin_dag_tool_resources", source) + self.assertNotIn("cluster_admin_baseline", source) + + def test_prod_uses_one_approved_orchestration_pipeline(self): + source = (ROOT / "modules/devops/cluster_admin_deploy_pipelines.tf").read_text( + encoding="utf-8" + ) + self.assertNotIn('resource "oci_devops_deploy_pipeline" "cluster_admin_approval"', source) + self.assertIn('resource "oci_devops_deploy_stage" "cluster_admin_approval"', source) + self.assertIn('resource "oci_devops_deploy_stage" "cluster_admin_orchestrator"', source) + self.assertIn( + "id = oci_devops_deploy_stage.cluster_admin_approval[each.key].id", + source, + ) + self.assertIn( + "id = oci_devops_deploy_pipeline.cluster_admin[each.key].id", + source, + ) + + def test_decommission_is_explicit_and_prod_requires_approval(self): + source = (ROOT / "modules/devops/cluster_admin_deploy_pipelines.tf").read_text( + encoding="utf-8" + ) + self.assertIn('display_name = "cluster-admin-${each.key}-decommission"', source) + self.assertIn( + "oci_devops_deploy_stage.cluster_admin_decommission_approval[each.key].id", + source, + ) + + command_spec = ( + ROOT / "templates/cluster-admin-decommission-command-spec.yaml.tpl" + ).read_text(encoding="utf-8") + self.assertNotIn("confirmation", source) + self.assertNotIn("confirmation", command_spec) + self.assertIn("oci devops deploy-stage get", command_spec) + self.assertIn('helm uninstall "$${tool_name}"', command_spec) + self.assertIn("Namespace $${namespace} was retained", command_spec) + + def test_operations_tags_are_consistent(self): + expected = { + "cluster": "noprod", + "owner": "cluster-administrators", + "purpose": "cluster-administration", + "role": "helm", + "scope": "operations", + "tool": "traefik", + } + dimensions = {"cluster": "noprod", "role": "helm", "tool": "traefik"} + self.assertEqual(dispatch.operation_tags(**dimensions), expected) + self.assertEqual(mirror.operation_tags(**dimensions), expected) + + def test_mirror_source_reference_supports_http_and_oci(self): + self.assertEqual( + mirror.source_reference("https://charts.example.test", "traefik", "mirror-traefik"), + "mirror-traefik/traefik", + ) + self.assertEqual( + mirror.source_reference( + "oci://registry-1.docker.io/bitnamicharts/", + "/external-dns", + "unused", + ), + "oci://registry-1.docker.io/bitnamicharts/external-dns", + ) + self.assertEqual( + mirror.target_chart_name("prometheus-community/kube-prometheus-stack"), + "kube-prometheus-stack", + ) + + def test_clusters_run_as_pipeline_deployments(self): + calls = [] + original = dispatch.oci + dispatch.oci = lambda arguments: calls.append(arguments) or { + "data": {"lifecycle-state": "SUCCEEDED"} + } + try: + for cluster in ("noprod", "prod"): + dispatch.run_pipeline( + f"{cluster}-pipeline", + cluster, + "a" * 40, + {"cluster_name": cluster, "config_commit": "a" * 40}, + "external-dns[helm], traefik[helm+resources]", + ) + finally: + dispatch.oci = original + + self.assertEqual(len(calls), 2) + for cluster, command in zip(("noprod", "prod"), calls): + self.assertIn("create-pipeline-deployment", command) + self.assertNotIn("create-single-stage-deployment", command) + arguments = json.loads(command[command.index("--deployment-arguments") + 1])["items"] + self.assertIn({"name": "cluster_name", "value": cluster}, arguments) + tags = json.loads(command[command.index("--freeform-tags") + 1]) + self.assertEqual(tags["cluster"], cluster) + self.assertEqual(tags["role"], "cluster-deployment") + + def test_empty_pipeline_defaults_use_nonempty_sentinel(self): + original = dispatch.oci + dispatch.oci = lambda arguments: { + "data": { + "deploy-pipeline-parameters": { + "items": [ + {"name": "config_commit", "default-value": ""}, + {"name": "cluster_id", "default-value": "cluster"}, + {"name": "cluster_name", "default-value": "noprod"}, + ] + } + } + } + try: + arguments = dispatch.pipeline_arguments("pipeline", "a" * 40) + finally: + dispatch.oci = original + self.assertEqual(arguments["config_commit"], "a" * 40) + self.assertEqual(arguments["cluster_id"], "cluster") + self.assertEqual(arguments["cluster_name"], "noprod") + + def test_independent_tools_share_a_wave(self): + targets = [ + {"tool": "cert-manager", "depends_on": []}, + {"tool": "metrics-server", "depends_on": []}, + {"tool": "traefik", "depends_on": ["cert-manager"]}, + ] + waves = [[target["tool"] for target in wave] for wave in dispatch.dependency_waves(targets)] + self.assertEqual(waves, [["cert-manager", "metrics-server"], ["traefik"]]) + orchestrator_waves = [ + [target["tool"] for target in wave] + for wave in cluster_deploy.dependency_waves(targets) + ] + self.assertEqual(orchestrator_waves, waves) + + def test_execution_plan_lists_waves_actions_and_baseline_last(self): + targets = [ + { + "actions": ["helm"], + "chart_version": "1.0.0", + "cluster": "prod", + "depends_on": [], + "kind": "tool", + "namespace": "external-dns", + "tool": "external-dns", + }, + { + "actions": ["helm", "resources"], + "chart_version": "2.0.0", + "cluster": "prod", + "depends_on": ["external-dns"], + "kind": "tool", + "namespace": "traefik", + "tool": "traefik", + }, + {"cluster": "prod", "kind": "baseline"}, + ] + plan = dispatch.cluster_plan("prod", targets, "a" * 40) + self.assertIn("Wave 1:\n - external-dns: helm", plan) + self.assertIn("Wave 2:\n - traefik: helm + resources", plan) + self.assertLess(plan.index("Wave 2"), plan.index("Final:")) + self.assertIn("values=" + "a" * 40, plan) + + def test_cluster_orchestrator_uses_immutable_values_and_atomic_helm(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + resources = root / "clusters/prod/tools/traefik/resources" + resources.mkdir(parents=True) + (resources / "configmap.yaml").write_text( + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: test\n", + encoding="utf-8", + ) + args = SimpleNamespace( + artifact_repository_id="artifact-repository", + chart_prefix="project/charts/cluster-tools", + cluster="prod", + commit="a" * 40, + registry="fra.ocir.io", + repo=root, + tenancy_namespace="namespace", + ) + target = { + "actions": ["helm", "resources"], + "chart": "traefik", + "chart_version": "1.2.3", + "depends_on": [], + "namespace": "traefik", + "tool": "traefik", + } + calls = [] + original = cluster_deploy.run + cluster_deploy.run = lambda command, **kwargs: calls.append(command) + try: + cluster_deploy.deploy_tool( + args, + target, + root, + ) + finally: + cluster_deploy.run = original + + download, helm, resources_apply, status = calls + self.assertIn("cluster-admin/prod/tools/traefik/values.yaml", download) + self.assertIn("a" * 40, download) + self.assertIn("oci://fra.ocir.io/namespace/project/charts/cluster-tools/traefik", helm) + self.assertIn("--atomic", helm) + self.assertIn("--version", helm) + self.assertEqual(helm[helm.index("--version") + 1], "1.2.3") + self.assertEqual(resources_apply[0:2], ["kubectl", "apply"]) + self.assertEqual(status[0:2], ["helm", "status"]) + + def test_cluster_command_consumes_immutable_plan_and_applies_baseline_last(self): + command_spec = ( + ROOT / "templates/cluster-admin-deploy-command-spec.yaml.tpl" + ).read_text(encoding="utf-8") + deploy_script = ( + ROOT / "repos/cluster-admin/script/deploy-cluster.py" + ).read_text(encoding="utf-8") + + self.assertIn("cluster-admin/deployment-plan.json", command_spec) + self.assertNotIn("--mode all", command_spec) + self.assertNotIn("pip install", command_spec) + self.assertNotIn("import yaml", deploy_script) + self.assertLess( + deploy_script.index("for number, wave in enumerate"), + deploy_script.index('if any(target["kind"] == "baseline"'), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_developer_templates.py b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_developer_templates.py new file mode 100644 index 000000000..53ab8e71e --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/tests/test_developer_templates.py @@ -0,0 +1,241 @@ +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class DeveloperTemplateTests(unittest.TestCase): + def read(self, relative): + return (ROOT / relative).read_text(encoding="utf-8") + + def resource_body(self, resource_name): + source = self.read("modules/devops/artifacts.tf") + start = source.index( + f'resource "oci_devops_deploy_artifact" "{resource_name}" {{' + ) + next_resource = source.find('\nresource "', start + 1) + return source[start:] if next_resource == -1 else source[start:next_resource] + + def test_release_command_specs_are_shared_singletons(self): + for resource_name in ( + "promote_release_image_command_spec", + "tag_release_commit_command_spec", + ): + self.assertNotIn("for_each", self.resource_body(resource_name)) + + def test_component_build_specs_export_generic_deployment_parameters(self): + for template in ( + "templates/application-delivery-pipeline.yaml.tpl", + "templates/release-pipeline.yaml.tpl", + ): + source = self.read(template) + self.assertIn("- image_repository", source) + self.assertIn("- image_tag", source) + self.assertNotRegex(source, r"component_image_(repository|tag)_variable") + + def test_production_values_keep_rc_parameter_for_chart_derivation(self): + source = self.resource_body("component_prod_values") + self.assertRegex(source, re.compile(r'image_tag_parameter\s*=\s*"image_tag"')) + + deployment = self.read("templates/component-chart-deployment.yaml.tpl") + self.assertIn('regexReplaceAll "-rc\\\\.[0-9]+$" .Values.image.tag ""', deployment) + + def test_component_release_approval_follows_staging(self): + source = self.read("modules/devops/deploy_pipelines.tf") + self.assertNotIn( + 'resource "oci_devops_deploy_stage" "dry_run_component_prod"', source + ) + self.assertNotIn('name = "DRY_RUN"', source) + self.assertIn( + 'id = oci_devops_deploy_stage.deploy_component["${each.key}:staging"].id', + source, + ) + self.assertIn("Approve promotion of ${each.value.name} from staging to production", source) + + def test_promotion_exports_generic_final_tag(self): + source = self.read("templates/promote-release-image-command-spec.yaml.tpl") + self.assertIn("- release_image_tag", source) + self.assertIn('release_image_tag="$${release_candidate_tag%-rc.*}"', source) + + def test_development_mode_refreshes_generated_pipeline_specs(self): + source = self.read("modules/devops/repositories.tf") + self.assertIn('resource "null_resource" "refresh_platform_development"', source) + self.assertIn('SEED_MODE = "refresh"', source) + self.assertIn("local.generated_component_build_specs", source) + self.assertIn("component.build_spec_path", source) + self.assertIn('"${name}-release-pipeline.yaml"', source) + self.assertIn( + 'fileset("${path.root}/${local.platform_repo_path}/script", "**")', + source, + ) + self.assertIn("filesha256", source) + + def test_repository_starters_survive_oci_initial_commits(self): + source = self.read("modules/devops/repositories.tf") + + for resource_name in ( + "seed_platform_shared", + "seed_application_source", + "seed_application_chart_baseline", + ): + start = source.index(f'resource "null_resource" "{resource_name}" {{') + next_resource = source.find('\nresource "', start + 1) + body = source[start:] if next_resource == -1 else source[start:next_resource] + self.assertRegex(body, r'SEED_MODE\s*=\s*"add-only"') + self.assertIn('seed_mode', body) + self.assertNotRegex(body, r'SEED_MODE\s*=\s*"empty-repository"') + + def test_pipeline_wrappers_do_not_require_executable_git_modes(self): + image_wrapper = self.read( + "repos/pipelines/script/application-delivery-build-image.sh" + ) + chart_wrapper = self.read( + "repos/pipelines/script/application-delivery-package-chart.sh" + ) + + self.assertIn('bash "${SCRIPT_DIR}/build-push-image.sh"', image_wrapper) + self.assertIn('bash "${SCRIPT_DIR}/package-push-chart.sh"', chart_wrapper) + + def test_custom_component_build_specs_are_referenced_but_not_generated(self): + locals_source = self.read("modules/devops/locals.tf") + self.assertIn( + 'build_spec_path = coalesce(component.build_spec_path, "${component.name}-build-pipeline.yaml")', + locals_source, + ) + self.assertIn( + "generated_component_build_specs", locals_source + ) + self.assertIn("if component.generate_build_spec", locals_source) + + build_pipelines = self.read("modules/devops/build_pipelines.tf") + self.assertIn("build_spec_file = each.value.build_spec_path", build_pipelines) + + repositories = self.read("modules/devops/repositories.tf") + self.assertIn( + "for_each = local.generated_component_build_specs", repositories + ) + self.assertIn( + "local.generated_component_build_specs[name].build_spec_path", + repositories, + ) + self.assertIn( + 'resource "local_file" "custom_application_delivery_pipeline"', + repositories, + ) + self.assertIn("sort(local.custom_build_spec_paths)", repositories) + self.assertIn("custom_build_spec_paths", repositories) + self.assertIn("never refreshed by Resource Manager", repositories) + + self.assertIn("custom_build_specs_by_path", locals_source) + self.assertIn("component_names = sort", locals_source) + + def test_application_bootstrap_initializes_both_clusters_in_parallel(self): + source = self.read("modules/devops/deploy_pipelines.tf") + self.assertIn('display_name = "${each.value.name}-bootstrap"', source) + self.assertIn( + "for_each = local.application_bootstrap_targets", + source, + ) + self.assertIn( + "command_spec_deploy_artifact_id = oci_devops_deploy_artifact.application_bootstrap_command_spec.id", + source, + ) + self.assertNotIn('resource "oci_devops_deploy_pipeline" "namespace_init"', source) + self.assertNotIn('resource "oci_devops_deploy_stage" "application_bootstrap_baseline"', source) + + def test_application_bootstrap_credentials_default_to_empty(self): + source = self.read("modules/devops/deploy_pipelines.tf") + bootstrap_start = source.index( + 'resource "oci_devops_deploy_pipeline" "application_bootstrap"' + ) + bootstrap_end = source.index( + 'resource "oci_devops_deploy_stage" "application_bootstrap_namespace"' + ) + bootstrap = source[bootstrap_start:bootstrap_end] + + for parameter_name in ("registry_username", "pull_password_secret_ocid"): + parameter_start = bootstrap.index(f'name = "{parameter_name}"') + parameter = bootstrap[parameter_start:parameter_start + 200] + self.assertIn('default_value = null', parameter) + + self.assertNotIn('default_value = "0.1.0"', bootstrap) + + locals_source = self.read("modules/devops/locals.tf") + self.assertIn('"${application_name}:noprod"', locals_source) + self.assertIn('"${application_name}:prod"', locals_source) + + command_spec = self.read( + "templates/application-bootstrap-command-spec.yaml.tpl" + ) + self.assertIn( + 'oci devops deploy-stage get --stage-id "$${OCI_STAGE_ID}"', + command_spec, + ) + self.assertNotIn("--deploy-stage-id", command_spec) + + def test_component_verification_runs_helm_status_in_prod_only(self): + source = self.read("modules/devops/deploy_pipelines.tf") + self.assertIn('resource "oci_devops_deploy_stage" "verify_component_prod"', source) + self.assertNotIn('resource "oci_devops_deploy_stage" "verify_component"', source) + self.assertIn( + "command_spec_deploy_artifact_id = oci_devops_deploy_artifact.component_verify_deployment_command_spec.id", + source, + ) + self.assertIn( + 'id = oci_devops_deploy_stage.deploy_component["${each.key}:staging"].id', + source, + ) + self.assertIn( + "id = oci_devops_deploy_stage.tag_component_release_commit[each.key].id", + source, + ) + self.assertIn( + "id = oci_devops_deploy_stage.deploy_component_prod[each.key].id", + source, + ) + + command_spec = self.read( + "templates/verify-component-production-command-spec.yaml.tpl" + ) + self.assertIn('helm status "$${release}"', command_spec) + self.assertIn('helm history "$${release}"', command_spec) + self.assertIn('helm get notes "$${release}"', command_spec) + self.assertIn('helm list --namespace "$${namespace}"', command_spec) + self.assertNotIn("helm get manifest", command_spec) + self.assertNotIn("helm get values", command_spec) + self.assertIn('prod_cluster_id: "${prod_oke_cluster_id}"', command_spec) + self.assertNotIn("verify-deployment.sh", command_spec) + self.assertIn( + 'oci devops deploy-stage get --stage-id "$${OCI_STAGE_ID}"', + command_spec, + ) + self.assertNotIn("--deploy-stage-id", command_spec) + + def test_application_input_validates_derived_names_and_collisions(self): + variables = self.read("variables.tf") + self.assertIn("Applications are limited to 46 characters", variables) + self.assertIn("components to 45", variables) + self.assertIn("Application namespaces must be unique", variables) + self.assertIn("Derived repository names must be unique", variables) + self.assertIn("Chart repository names and paths must use safe lowercase relative naming", variables) + + checks = self.read("checks.tf") + self.assertIn('check "recommended_application_scale"', checks) + self.assertIn("local.component_count <= 50", checks) + + def test_deployment_parameter_descriptions_do_not_use_html_placeholders(self): + for path in (ROOT / "modules").rglob("*.tf"): + source = path.read_text(encoding="utf-8") + for description in re.findall(r'description\s*=\s*"([^"]*)"', source): + self.assertNotRegex(description, r"<[^>]+>", path) + + def test_release_archive_excludes_maintainer_agent_material(self): + packaging = self.read("update.sh") + self.assertIn('--exclude ".agents"', packaging) + self.assertIn('--exclude "AGENT.md"', packaging) + + +if __name__ == "__main__": + unittest.main() diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/update.sh b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/update.sh new file mode 100755 index 000000000..e328c137c --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/update.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_ZIP="${STACK_ZIP_PATH:-${ROOT_DIR}/stack.zip}" +DEVELOPMENT_MODE="${STACK_DEVELOPMENT_MODE:-false}" + +if [ "$DEVELOPMENT_MODE" != "true" ] && [ "$DEVELOPMENT_MODE" != "false" ]; then + echo "STACK_DEVELOPMENT_MODE must be true or false." >&2 + exit 1 +fi + +STAGING_DIR="$(mktemp -d)" +cleanup() { + rm -rf "$STAGING_DIR" +} +trap cleanup EXIT + +rsync -a \ + --exclude ".git" \ + --exclude ".agents" \ + --exclude ".terraform" \ + --exclude ".idea" \ + --exclude ".oca" \ + --exclude ".tmp-*" \ + --exclude "__pycache__" \ + --exclude "*.pyc" \ + --exclude "AGENT.md" \ + --exclude "script/update_orm_stack.sh" \ + --exclude "terraform.tfstate*" \ + --exclude "*.tfvars" \ + --exclude ".DS_Store" \ + --exclude "stack.zip" \ + "$ROOT_DIR/" "$STAGING_DIR/" + +if [ "$DEVELOPMENT_MODE" = "true" ]; then + find "$STAGING_DIR" -name '*.tf' -type f -exec perl -0pi -e \ + 's/\n[ \t]*lifecycle \{\n[ \t]*ignore_changes = all\n[ \t]*\}\n/\n/g' {} + + printf 'development_mode = true\n' >"$STAGING_DIR/development.auto.tfvars" +fi + +rm -f "$OUTPUT_ZIP" +( + cd "$STAGING_DIR" + zip -qr "$OUTPUT_ZIP" . +) + +printf "Built %s in %s mode\n" "$OUTPUT_ZIP" "$([ "$DEVELOPMENT_MODE" = "true" ] && echo development || echo release)" diff --git a/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/variables.tf b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/variables.tf new file mode 100644 index 000000000..96661c04a --- /dev/null +++ b/oci-and-db/cloud-native/devops-and-containers/devops/oci-devops-rm/variables.tf @@ -0,0 +1,540 @@ +variable "region" {} +variable "tenancy_ocid" {} +variable "current_user_ocid" {} +variable "compartment_ocid" {} + +variable "devops_compartment_id" { + default = null +} + +variable "devops_project_name" { + default = "oke-devops-starter" + + validation { + condition = ( + trimspace(var.devops_project_name) != "" && + replace(lower(var.devops_project_name), "/[^a-z0-9._-]+/", "-") != "" && + length(var.devops_project_name) <= 100 + ) + error_message = "devops_project_name must be nonempty, produce a valid image/chart prefix, and contain at most 100 characters." + } +} + +variable "devops_project_description" { + default = "Beginner-friendly OCI DevOps project for OKE Helm delivery" +} + +variable "devops_log_group_name" { + default = "oke-helm-starter-logs" +} + +variable "devops_log_group_description" { + default = "OCI DevOps logs for the OKE Helm starter project" +} + +variable "devops_log_name" { + default = null +} + +variable "devops_log_is_enabled" { + type = bool + default = true +} + +variable "devops_log_retention_period_in_days" { + type = number + default = 30 +} + +variable "create_notification_topic" { + type = bool + default = true +} + +variable "notification_topic_id" { + default = null +} + +variable "notification_topic_name" { + default = "oke-helm-starter-topic" +} + +variable "notification_topic_description" { + default = "OCI DevOps notifications for the OKE Helm starter project" +} + +variable "oke_compartment_id" { + default = null +} + +variable "oke_cluster_id" {} + +variable "prod_oke_cluster_id" { + default = null + + validation { + condition = try(trimspace(var.prod_oke_cluster_id), "") != "" + error_message = "prod_oke_cluster_id is required for the production OKE environment." + } +} + +variable "prod_oke_compartment_id" { + default = null + + validation { + condition = try(trimspace(var.prod_oke_compartment_id), "") != "" + error_message = "prod_oke_compartment_id is required for the production OKE cluster." + } +} + +variable "network_compartment_id" { + default = null +} + +variable "prod_network_compartment_id" { + default = null + + validation { + condition = try(trimspace(var.prod_network_compartment_id), "") != "" + error_message = "prod_network_compartment_id is required for the production OKE network configuration." + } +} + +variable "oke_vcn_id" { + default = null +} + +variable "prod_oke_vcn_id" { + default = null + + validation { + condition = try(trimspace(var.prod_oke_vcn_id), "") != "" + error_message = "prod_oke_vcn_id is required for the production OKE network configuration." + } +} + +variable "oke_worker_subnet_id" { + default = null + + validation { + condition = try(trimspace(var.oke_worker_subnet_id), "") != "" + error_message = "oke_worker_subnet_id is required for application bootstrap shell stage execution." + } +} + +variable "oke_worker_nsg_id" { + default = null +} + +variable "prod_oke_worker_subnet_id" { + default = null + + validation { + condition = try(trimspace(var.prod_oke_worker_subnet_id), "") != "" + error_message = "prod_oke_worker_subnet_id is required for production private endpoint and shell stage execution." + } +} + +variable "prod_oke_worker_nsg_id" { + default = null +} + +variable "applications" { + description = "JSON array of applications and globally unique components to bootstrap in OCI DevOps." + type = string + default = <<-JSON + [ + { + "name": "sample-app", + "components": [ + { + "name": "sample-api" + }, + { + "name": "sample-worker" + } + ] + } + ] + JSON + + validation { + condition = can([ + for application in jsondecode(var.applications) : { + name = application.name + components = [ + for component in application.components : component.name + ] + } + ]) + error_message = "applications must be a JSON array whose entries contain name and a components array with component names." + } + + validation { + condition = can(length(jsondecode(var.applications)) > 0) ? length(jsondecode(var.applications)) > 0 : true + error_message = "applications must contain at least one application." + } + + validation { + condition = can(alltrue(flatten([ + for application in jsondecode(var.applications) : concat( + [can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", application.name)) && length(application.name) <= 46], + [for component in application.components : can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", component.name)) && length(component.name) <= 45] + ) + ]))) ? alltrue(flatten([ + for application in jsondecode(var.applications) : concat( + [can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", application.name)) && length(application.name) <= 46], + [for component in application.components : can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", component.name)) && length(component.name) <= 45] + ) + ])) : true + error_message = "Application and component names must be Kubernetes DNS labels. Applications are limited to 46 characters and components to 45 so derived Helm release names remain valid." + } + + validation { + condition = can([for application in jsondecode(var.applications) : application.name]) ? length(distinct([ + for application in jsondecode(var.applications) : application.name + ])) == length(jsondecode(var.applications)) : true + error_message = "Application names must be unique." + } + + validation { + condition = can(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : component.name + ] + ])) ? length(distinct(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : component.name + ] + ]))) == length(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : component.name + ] + ])) : true + error_message = "Component names must be globally unique across all applications." + } + + validation { + condition = can(alltrue([ + for application in jsondecode(var.applications) : length(application.components) > 0 + ])) ? alltrue([ + for application in jsondecode(var.applications) : length(application.components) > 0 + ]) : true + error_message = "Every application must contain at least one component." + } + + validation { + condition = can(alltrue([ + for application in jsondecode(var.applications) : + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(application.namespace, application.name))) && + length(try(application.namespace, application.name)) <= 63 && + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(application.prod_namespace, try(application.namespace, application.name)))) && + length(try(application.prod_namespace, try(application.namespace, application.name))) <= 63 + ])) ? alltrue([ + for application in jsondecode(var.applications) : + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(application.namespace, application.name))) && + length(try(application.namespace, application.name)) <= 63 && + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(application.prod_namespace, try(application.namespace, application.name)))) && + length(try(application.prod_namespace, try(application.namespace, application.name))) <= 63 + ]) : true + error_message = "Application noprod and prod namespaces must be Kubernetes DNS labels of at most 63 characters." + } + + validation { + condition = can(jsondecode(var.applications)) ? ( + length(distinct([ + for application in jsondecode(var.applications) : try(application.namespace, application.name) + ])) == length(jsondecode(var.applications)) && + length(distinct([ + for application in jsondecode(var.applications) : try(application.prod_namespace, try(application.namespace, application.name)) + ])) == length(jsondecode(var.applications)) + ) : true + error_message = "Application namespaces must be unique within both the noprod and prod clusters." + } + + validation { + condition = can(jsondecode(var.applications)) ? length(distinct(concat( + ["pipelines", "cluster-admin"], + [for application in jsondecode(var.applications) : try(application.chart_repository_name, "${application.name}-chart")], + flatten([for application in jsondecode(var.applications) : [for component in application.components : component.name]]) + ))) == length(concat( + ["pipelines", "cluster-admin"], + [for application in jsondecode(var.applications) : try(application.chart_repository_name, "${application.name}-chart")], + flatten([for application in jsondecode(var.applications) : [for component in application.components : component.name]]) + )) : true + error_message = "Derived repository names must be unique and cannot collide with the reserved pipelines or cluster-admin repositories." + } + + validation { + condition = can(alltrue(flatten([ + for application in jsondecode(var.applications) : concat( + [ + can(regex("^[a-z0-9][a-z0-9._-]*[a-z0-9]$|^[a-z0-9]$", try(application.chart_repository_name, "${application.name}-chart"))) && + length(try(application.chart_repository_name, "${application.name}-chart")) <= 100 && + can(regex("^[a-z0-9][a-z0-9._/-]*[a-z0-9]$|^[a-z0-9]$", try(application.chart_path, application.name))) && + !startswith(try(application.chart_path, application.name), "/") && + !strcontains(try(application.chart_path, application.name), "..") && + length(try(application.chart_path, application.name)) <= 120 && + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+([-+][0-9A-Za-z.-]+)?$", try(application.chart_version, "0.1.0"))) + ], + [ + for component in application.components : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+([-+][0-9A-Za-z.-]+)?$", try(component.chart_version, "0.1.0"))) + ] + ) + ]))) ? alltrue(flatten([ + for application in jsondecode(var.applications) : concat( + [ + can(regex("^[a-z0-9][a-z0-9._-]*[a-z0-9]$|^[a-z0-9]$", try(application.chart_repository_name, "${application.name}-chart"))) && + length(try(application.chart_repository_name, "${application.name}-chart")) <= 100 && + can(regex("^[a-z0-9][a-z0-9._/-]*[a-z0-9]$|^[a-z0-9]$", try(application.chart_path, application.name))) && + !startswith(try(application.chart_path, application.name), "/") && + !strcontains(try(application.chart_path, application.name), "..") && + length(try(application.chart_path, application.name)) <= 120 && + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+([-+][0-9A-Za-z.-]+)?$", try(application.chart_version, "0.1.0"))) + ], + [ + for component in application.components : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+([-+][0-9A-Za-z.-]+)?$", try(component.chart_version, "0.1.0"))) + ] + ) + ])) : true + error_message = "Chart repository names and paths must use safe lowercase relative naming within their length limits, and chart versions must use SemVer." + } + + validation { + condition = can(alltrue(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : + can(regex("^[A-Za-z0-9._/-]+[.]ya?ml$", coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"))) && + !startswith(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "/") && + !startswith(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "script/") && + !strcontains(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "..") && + length(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml")) <= 255 + ] + ]))) ? alltrue(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : + can(regex("^[A-Za-z0-9._/-]+[.]ya?ml$", coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"))) && + !startswith(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "/") && + !startswith(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "script/") && + !strcontains(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml"), "..") && + length(coalesce(try(component.build_spec_path, null), "${component.name}-build-pipeline.yaml")) <= 255 + ] + ])) : true + error_message = "Component build_spec_path values must be safe relative .yaml or .yml paths outside the managed script directory." + } + + validation { + condition = can(alltrue(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : + !contains(concat( + ["README.md", "helm-chart-pipeline.yaml"], + [for configured_application in jsondecode(var.applications) : "${configured_application.name}-package-pipeline.yaml"], + flatten([for configured_application in jsondecode(var.applications) : [for configured_component in configured_application.components : "${configured_component.name}-release-pipeline.yaml"]]), + flatten([for configured_application in jsondecode(var.applications) : [for configured_component in configured_application.components : "${configured_component.name}-build-pipeline.yaml" if try(configured_component.build_spec_path, null) == null]]) + ), component.build_spec_path) if try(component.build_spec_path, null) != null + ] + ]))) ? alltrue(flatten([ + for application in jsondecode(var.applications) : [ + for component in application.components : + !contains(concat( + ["README.md", "helm-chart-pipeline.yaml"], + [for configured_application in jsondecode(var.applications) : "${configured_application.name}-package-pipeline.yaml"], + flatten([for configured_application in jsondecode(var.applications) : [for configured_component in configured_application.components : "${configured_component.name}-release-pipeline.yaml"]]), + flatten([for configured_application in jsondecode(var.applications) : [for configured_component in configured_application.components : "${configured_component.name}-build-pipeline.yaml" if try(configured_component.build_spec_path, null) == null]]) + ), component.build_spec_path) if try(component.build_spec_path, null) != null + ] + ])) : true + error_message = "Explicit component build_spec_path values cannot collide with stack-managed pipeline repository files." + } +} + +variable "enable_cluster_admin" { + description = "Create the optional OCI DevOps cluster-administration workflow." + type = bool + default = false +} + +variable "cluster_admin_artifact_repository_name" { + description = "Optional display name for the Generic Artifact repository containing cluster-admin values and production plans. Empty derives the name from the DevOps project." + type = string + default = "" + + validation { + condition = trimspace(var.cluster_admin_artifact_repository_name) == "" || ( + var.cluster_admin_artifact_repository_name == trimspace(var.cluster_admin_artifact_repository_name) && + length(var.cluster_admin_artifact_repository_name) <= 255 && + can(regex("^[A-Za-z0-9][A-Za-z0-9._ -]*$", var.cluster_admin_artifact_repository_name)) + ) + error_message = "cluster_admin_artifact_repository_name must be empty or a 1-255 character name beginning with a letter or number and containing only letters, numbers, spaces, periods, underscores, and hyphens." + } +} + +variable "cluster_administration" { + description = "JSON object configuring one shared Kubernetes tool topology and its pinned public Helm chart sources for the existing noprod and prod OKE clusters." + type = string + default = <<-JSON + { + "tools": [ + { + "name": "keda", + "repository": "https://kedacore.github.io/charts", + "chart": "keda", + "version": "2.20.1", + "namespace": "keda", + "depends_on": [] + }, + { + "name": "kube-prometheus", + "repository": "https://prometheus-community.github.io/helm-charts", + "chart": "kube-prometheus-stack", + "version": "87.10.1", + "namespace": "monitoring", + "depends_on": [] + } + ] + } + JSON + + validation { + condition = can([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : { + name = tool.name + repository = tool.repository + chart = tool.chart + version = tool.version + namespace = try(tool.namespace, tool.name) + depends_on = try(tool.depends_on, []) + } + ]) + error_message = "cluster_administration must contain a tools array whose entries define name, repository, chart, and version. Legacy input with noprod.tools is also accepted." + } + + validation { + condition = can(alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + can(regex("^(https|oci)://[^[:space:]]+$", tool.repository)) && + can(regex("^[A-Za-z0-9][A-Za-z0-9._/-]*$", tool.chart)) && + can(regex("^[0-9A-Za-z][0-9A-Za-z._+-]*$", tool.version)) + ])) ? alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + can(regex("^(https|oci)://[^[:space:]]+$", tool.repository)) && + can(regex("^[A-Za-z0-9][A-Za-z0-9._/-]*$", tool.chart)) && + can(regex("^[0-9A-Za-z][0-9A-Za-z._+-]*$", tool.version)) + ]) : true + error_message = "Each cluster tool must define an HTTPS Helm repository or OCI chart repository, a valid chart name, and a pinned chart version." + } + + validation { + condition = can(alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", tool.name)) && + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(tool.namespace, tool.name))) && + length(tool.name) <= 63 && length(try(tool.namespace, tool.name)) <= 63 + ])) ? alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", tool.name)) && + can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", try(tool.namespace, tool.name))) && + length(tool.name) <= 63 && length(try(tool.namespace, tool.name)) <= 63 + ]) : true + error_message = "Cluster tool names and namespaces must be valid Kubernetes DNS labels of at most 63 characters." + } + + validation { + condition = can( + length(distinct([for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : tool.name])) == + length(try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools)) + ) ? ( + length(distinct([for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : tool.name])) == + length(try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools)) + ) : true + error_message = "Tool names must be unique." + } + + validation { + condition = can( + length(distinct([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : try(tool.namespace, tool.name) + ])) == length([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : try(tool.namespace, tool.name) + ]) + ) ? ( + length(distinct([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : try(tool.namespace, tool.name) + ])) == length([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : try(tool.namespace, tool.name) + ]) + ) : true + error_message = "Each tool must use a unique namespace." + } + + validation { + condition = can(alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + alltrue([ + for dependency in try(tool.depends_on, []) : + dependency != tool.name && contains( + [for candidate in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : candidate.name], + dependency + ) + ]) && length(distinct(try(tool.depends_on, []))) == length(try(tool.depends_on, [])) + ])) ? alltrue([ + for tool in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : + alltrue([ + for dependency in try(tool.depends_on, []) : + dependency != tool.name && contains( + [for candidate in try(jsondecode(var.cluster_administration).tools, jsondecode(var.cluster_administration).noprod.tools) : candidate.name], + dependency + ) + ]) && length(distinct(try(tool.depends_on, []))) == length(try(tool.depends_on, [])) + ]) : true + error_message = "Tool dependencies must be unique, reference another configured tool, and cannot reference the tool itself." + } + +} + +variable "namespace_init_secret_name" { + default = "ocirsecret" + + validation { + condition = can(regex("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", var.namespace_init_secret_name)) && length(var.namespace_init_secret_name) <= 63 + error_message = "namespace_init_secret_name must be a Kubernetes DNS label of at most 63 characters." + } +} + +variable "namespace_init_secret_compartment_id" { + default = null +} + +variable "auth_token" { + sensitive = true +} + +variable "create_iam" { + type = bool + default = false +} + +variable "iam_domain_compartment_id" { + default = null +} + +variable "devops_iam_domain_id" { + default = null +} + +variable "devops_dynamic_group_name" { + default = "OkeHelmStarterDevOpsDynamicGroup" +} + +variable "devops_policy_name" { + default = "oke-helm-starter-devops-policy" +} + +variable "development_mode" { + description = "Internal mode used by development archives to refresh template-owned repository files." + type = bool + default = false +}