From bc8c906fcf9d10d86020c220cb1214bfb792ce0e Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 01:08:11 +0530 Subject: [PATCH 01/15] Add countly-hosted Argo bootstrap --- argocd/countly-hosted/README.md | 35 ++++++++ .../applicationsets/00-mongodb.yaml | 72 +++++++++++++++ .../applicationsets/01-clickhouse.yaml | 78 ++++++++++++++++ .../applicationsets/02-kafka.yaml | 90 +++++++++++++++++++ .../applicationsets/03-countly.yaml | 84 +++++++++++++++++ .../applicationsets/04-observability.yaml | 64 +++++++++++++ .../applicationsets/05-migration.yaml | 55 ++++++++++++ .../operators/00-cert-manager.yaml | 41 +++++++++ .../operators/01-mongodb-crds.yaml | 38 ++++++++ .../operators/02-mongodb-operator.yaml | 42 +++++++++ .../operators/03-clickhouse-operator.yaml | 41 +++++++++ .../operators/04-strimzi-operator.yaml | 38 ++++++++ .../operators/05-nginx-ingress.yaml | 79 ++++++++++++++++ .../06-letsencrypt-prod-issuer-app.yaml | 37 ++++++++ .../07-external-secrets-operator.yaml | 44 +++++++++ .../operators/08-cluster-secret-store.yaml | 48 ++++++++++ argocd/countly-hosted/projects/customers.yaml | 36 ++++++++ argocd/countly-hosted/root-application.yaml | 23 +++++ 18 files changed, 945 insertions(+) create mode 100644 argocd/countly-hosted/README.md create mode 100644 argocd/countly-hosted/applicationsets/00-mongodb.yaml create mode 100644 argocd/countly-hosted/applicationsets/01-clickhouse.yaml create mode 100644 argocd/countly-hosted/applicationsets/02-kafka.yaml create mode 100644 argocd/countly-hosted/applicationsets/03-countly.yaml create mode 100644 argocd/countly-hosted/applicationsets/04-observability.yaml create mode 100644 argocd/countly-hosted/applicationsets/05-migration.yaml create mode 100644 argocd/countly-hosted/operators/00-cert-manager.yaml create mode 100644 argocd/countly-hosted/operators/01-mongodb-crds.yaml create mode 100644 argocd/countly-hosted/operators/02-mongodb-operator.yaml create mode 100644 argocd/countly-hosted/operators/03-clickhouse-operator.yaml create mode 100644 argocd/countly-hosted/operators/04-strimzi-operator.yaml create mode 100644 argocd/countly-hosted/operators/05-nginx-ingress.yaml create mode 100644 argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml create mode 100644 argocd/countly-hosted/operators/07-external-secrets-operator.yaml create mode 100644 argocd/countly-hosted/operators/08-cluster-secret-store.yaml create mode 100644 argocd/countly-hosted/projects/customers.yaml create mode 100644 argocd/countly-hosted/root-application.yaml diff --git a/argocd/countly-hosted/README.md b/argocd/countly-hosted/README.md new file mode 100644 index 0000000..846dd9f --- /dev/null +++ b/argocd/countly-hosted/README.md @@ -0,0 +1,35 @@ +# Countly-Hosted Argo Bootstrap + +This path is the Countly-managed GitOps lane. + +It is intentionally separate from the public self-hosted Argo flow under `argocd/`. + +## What It Does + +- reads hosted customer metadata from the private `countly-deployment` repository +- deploys shared charts from the public `helm` repository +- combines public profiles with private customer value files through Argo CD multi-source applications + +## Repository Split + +- public `helm` repository + - charts + - profiles + - this hosted bootstrap +- private `countly-deployment` repository + - `customers/*.yaml` + - `environments//...` + +## Why This Exists + +This keeps: + +- shared product code public +- customer inventory private +- the hosted deployment path separate from the public self-hosted path + +## Main Entry Point + +- `root-application.yaml` + +Point a bootstrap `Application` at `argocd/countly-hosted` when Argo CD should manage Countly-hosted customers. diff --git a/argocd/countly-hosted/applicationsets/00-mongodb.yaml b/argocd/countly-hosted/applicationsets/00-mongodb.yaml new file mode 100644 index 0000000..8818e94 --- /dev/null +++ b/argocd/countly-hosted/applicationsets/00-mongodb.yaml @@ -0,0 +1,72 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-mongodb + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-mongodb + helm: + releaseName: countly-mongodb + valueFiles: + - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" + - "../../profiles/security/{{ .security }}/mongodb.yaml" + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/mongodb.yaml" + - "$values/environments/{{ .environment }}/credentials-mongodb.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: mongodbcommunity.mongodb.com + kind: MongoDBCommunity + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml new file mode 100644 index 0000000..6e384b0 --- /dev/null +++ b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml @@ -0,0 +1,78 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-clickhouse + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-clickhouse" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-clickhouse + helm: + releaseName: countly-clickhouse + valueFiles: + - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" + - "../../profiles/security/{{ .security }}/clickhouse.yaml" + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/clickhouse.yaml" + - "$values/environments/{{ .environment }}/credentials-clickhouse.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: clickhouse + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: clickhouse.com + kind: ClickHouseCluster + jsonPointers: + - /status + - group: clickhouse.com + kind: KeeperCluster + jsonPointers: + - /status + - /spec/containerTemplate/resources/requests/memory + - /spec/containerTemplate/resources/limits/memory + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml new file mode 100644 index 0000000..1ad3ef2 --- /dev/null +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -0,0 +1,90 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-kafka + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-kafka" + annotations: + argocd.argoproj.io/sync-wave: "5" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-kafka + helm: + releaseName: countly-kafka + valueFiles: + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" + - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" + - "../../profiles/observability/{{ .observability }}/kafka.yaml" + - "../../profiles/security/{{ .security }}/kafka.yaml" + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/kafka.yaml" + - "$values/environments/{{ .environment }}/credentials-kafka.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + - name: global.kafkaConnect + value: "{{ .kafkaConnect }}" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: kafka + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: kafka.strimzi.io + kind: Kafka + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnect + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnector + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaNodePool + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/countly-hosted/applicationsets/03-countly.yaml b/argocd/countly-hosted/applicationsets/03-countly.yaml new file mode 100644 index 0000000..a46998e --- /dev/null +++ b/argocd/countly-hosted/applicationsets/03-countly.yaml @@ -0,0 +1,84 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-app + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-countly" + annotations: + argocd.argoproj.io/sync-wave: "10" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly + helm: + releaseName: countly + valueFiles: + - "../../profiles/sizing/{{ .sizing }}/countly.yaml" + - "../../profiles/tls/{{ .tls }}/countly.yaml" + - "../../profiles/observability/{{ .observability }}/countly.yaml" + - "../../profiles/security/{{ .security }}/countly.yaml" + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/countly.yaml" + - "$values/environments/{{ .environment }}/credentials-countly.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: ingress.hostname + value: "{{ .hostname }}" + - name: ingress.tls.mode + value: '{{ if eq .tls "none" }}http{{ else if eq .tls "provided" }}existingSecret{{ else }}{{ .tls }}{{ end }}' + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + - name: global.tls + value: "{{ .tls }}" + - name: global.kafkaConnect + value: "{{ .kafkaConnect }}" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: countly + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: networking.k8s.io + kind: Ingress + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/countly-hosted/applicationsets/04-observability.yaml b/argocd/countly-hosted/applicationsets/04-observability.yaml new file mode 100644 index 0000000..0b6f3ef --- /dev/null +++ b/argocd/countly-hosted/applicationsets/04-observability.yaml @@ -0,0 +1,64 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-observability + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-observability" + annotations: + argocd.argoproj.io/sync-wave: "15" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .observability "disabled" }}charts/noop{{ else }}charts/countly-observability{{ end }}' + helm: + releaseName: countly-observability + valueFiles: + - "../../profiles/sizing/{{ .sizing }}/observability.yaml" + - "../../profiles/observability/{{ .observability }}/observability.yaml" + - "../../profiles/security/{{ .security }}/observability.yaml" + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/observability.yaml" + - "$values/environments/{{ .environment }}/credentials-observability.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: observability + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/argocd/countly-hosted/applicationsets/05-migration.yaml b/argocd/countly-hosted/applicationsets/05-migration.yaml new file mode 100644 index 0000000..de2f277 --- /dev/null +++ b/argocd/countly-hosted/applicationsets/05-migration.yaml @@ -0,0 +1,55 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-migration + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-migration" + annotations: + argocd.argoproj.io/sync-wave: "10" + spec: + project: "{{ .project }}" + sources: + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .migration "enabled" }}charts/countly-migration{{ else }}charts/noop{{ end }}' + helm: + releaseName: countly-migration + valueFiles: + - "$values/environments/{{ .environment }}/global.yaml" + - "$values/environments/{{ .environment }}/migration.yaml" + - "$values/environments/{{ .environment }}/credentials-migration.yaml" + parameters: + - name: argocd.enabled + value: "true" + - repoURL: https://github.com/Countly/countly-deployment.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: countly-migration + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/argocd/countly-hosted/operators/00-cert-manager.yaml b/argocd/countly-hosted/operators/00-cert-manager.yaml new file mode 100644 index 0000000..682eada --- /dev/null +++ b/argocd/countly-hosted/operators/00-cert-manager.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-cert-manager + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-cert-manager" + annotations: + argocd.argoproj.io/sync-wave: "-30" + spec: + project: default + source: + repoURL: https://charts.jetstack.io + chart: cert-manager + targetRevision: v1.17.2 + helm: + releaseName: cert-manager + parameters: + - name: installCRDs + value: "true" + destination: + server: "{{ .server }}" + namespace: cert-manager + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/01-mongodb-crds.yaml b/argocd/countly-hosted/operators/01-mongodb-crds.yaml new file mode 100644 index 0000000..3a90c1a --- /dev/null +++ b/argocd/countly-hosted/operators/01-mongodb-crds.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-mongodb-crds + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb-crds" + annotations: + argocd.argoproj.io/sync-wave: "-29" + spec: + project: default + source: + repoURL: https://github.com/mongodb/mongodb-kubernetes.git + targetRevision: "1.7.0" + path: public + directory: + include: crds.yaml + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/02-mongodb-operator.yaml b/argocd/countly-hosted/operators/02-mongodb-operator.yaml new file mode 100644 index 0000000..89a7261 --- /dev/null +++ b/argocd/countly-hosted/operators/02-mongodb-operator.yaml @@ -0,0 +1,42 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-mongodb-kubernetes-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb-kubernetes-operator" + annotations: + argocd.argoproj.io/sync-wave: "-28" + spec: + project: default + source: + repoURL: https://mongodb.github.io/helm-charts + chart: mongodb-kubernetes + targetRevision: 1.7.0 + helm: + releaseName: mongodb-kubernetes-operator + valuesObject: + operator: + watchedResources: + - mongodbcommunity + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml new file mode 100644 index 0000000..d3a6212 --- /dev/null +++ b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-clickhouse-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-clickhouse-operator" + annotations: + argocd.argoproj.io/sync-wave: "-27" + spec: + project: default + source: + repoURL: ghcr.io/clickhouse + chart: clickhouse-operator-helm + targetRevision: 0.0.2 + helm: + releaseName: clickhouse-operator + valuesObject: + certManager: + install: false + destination: + server: "{{ .server }}" + namespace: clickhouse-operator-system + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/04-strimzi-operator.yaml b/argocd/countly-hosted/operators/04-strimzi-operator.yaml new file mode 100644 index 0000000..51673cd --- /dev/null +++ b/argocd/countly-hosted/operators/04-strimzi-operator.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-strimzi-kafka-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-strimzi-kafka-operator" + annotations: + argocd.argoproj.io/sync-wave: "-26" + spec: + project: default + source: + repoURL: https://strimzi.io/charts/ + chart: strimzi-kafka-operator + targetRevision: 0.51.0 + helm: + releaseName: strimzi-kafka-operator + destination: + server: "{{ .server }}" + namespace: kafka + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/05-nginx-ingress.yaml b/argocd/countly-hosted/operators/05-nginx-ingress.yaml new file mode 100644 index 0000000..8f55a4a --- /dev/null +++ b/argocd/countly-hosted/operators/05-nginx-ingress.yaml @@ -0,0 +1,79 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-nginx-ingress + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-nginx-ingress" + annotations: + argocd.argoproj.io/sync-wave: "-25" + spec: + project: default + sources: + - repoURL: https://helm.nginx.com/stable + chart: nginx-ingress + targetRevision: 2.1.0 + helm: + releaseName: nginx-ingress + valueFiles: + - $values/nginx-ingress-values.yaml + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: ingress-nginx + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + ignoreDifferences: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apdoslogconfs.appprotectdos.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apdospolicies.appprotectdos.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: aplogconfs.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: appolicies.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apusersigs.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: "" + kind: Service + name: nginx-ingress-controller + namespace: ingress-nginx + jsonPointers: + - /metadata/annotations/cloud.google.com~1neg + - /spec/healthCheckNodePort + - /spec/ports/0/nodePort + - /spec/ports/1/nodePort diff --git a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml new file mode 100644 index 0000000..2c1a78c --- /dev/null +++ b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml @@ -0,0 +1,37 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-letsencrypt-prod-issuer + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-letsencrypt-prod-issuer" + annotations: + argocd.argoproj.io/sync-wave: "-24" + spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .tls "letsencrypt" }}argocd/operator-manifests/letsencrypt-prod-issuer{{ else }}charts/noop{{ end }}' + directory: + recurse: true + destination: + server: "{{ .server }}" + namespace: cert-manager + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml new file mode 100644 index 0000000..7d48bc5 --- /dev/null +++ b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-external-secrets + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-external-secrets" + annotations: + argocd.argoproj.io/sync-wave: "-23" + spec: + project: default + source: + repoURL: https://charts.external-secrets.io + chart: external-secrets + targetRevision: 1.3.1 + helm: + releaseName: external-secrets + values: | + installCRDs: true + serviceAccount: + create: true + annotations: + iam.gke.io/gcp-service-account: "{{ .gcpServiceAccountEmail }}" + destination: + server: "{{ .server }}" + namespace: external-secrets + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml new file mode 100644 index 0000000..db32a30 --- /dev/null +++ b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml @@ -0,0 +1,48 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-cluster-secret-store + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/countly-deployment.git + revision: main + files: + - path: customers/*.yaml + template: + metadata: + name: "{{ .customer }}-cluster-secret-store" + annotations: + argocd.argoproj.io/sync-wave: "-22" + spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-cluster-secret-store + helm: + releaseName: countly-cluster-secret-store + parameters: + - name: secretStore.name + value: "gcp-secrets" + - name: secretStore.secretManagerProjectID + value: "{{ .secretManagerProjectID }}" + - name: secretStore.clusterProjectID + value: "{{ .clusterProjectID }}" + - name: secretStore.clusterName + value: "{{ .clusterName }}" + - name: secretStore.clusterLocation + value: "{{ .clusterLocation }}" + destination: + server: "{{ .server }}" + namespace: external-secrets + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/argocd/countly-hosted/projects/customers.yaml b/argocd/countly-hosted/projects/customers.yaml new file mode 100644 index 0000000..d48d4f7 --- /dev/null +++ b/argocd/countly-hosted/projects/customers.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: countly-customers + namespace: argocd +spec: + description: "Shared AppProject for GitOps-managed Countly customer environments" + sourceRepos: + - '*' + destinations: + - server: '*' + namespace: mongodb + - server: '*' + namespace: clickhouse + - server: '*' + namespace: kafka + - server: '*' + namespace: countly + - server: '*' + namespace: observability + - server: '*' + namespace: countly-migration + clusterResourceWhitelist: + - group: "" + kind: Namespace + - group: storage.k8s.io + kind: StorageClass + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: cert-manager.io + kind: ClusterIssuer + namespaceResourceWhitelist: + - group: '*' + kind: '*' diff --git a/argocd/countly-hosted/root-application.yaml b/argocd/countly-hosted/root-application.yaml new file mode 100644 index 0000000..ffce1cc --- /dev/null +++ b/argocd/countly-hosted/root-application.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: countly-hosted-bootstrap + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: argocd/countly-hosted + directory: + recurse: true + exclude: "{operator-manifests/**}" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true From acda4beb51b91fd4e6b51f8c8347725f8200b7b3 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 11:18:23 +0530 Subject: [PATCH 02/15] Point hosted bootstrap to test branches --- argocd/countly-hosted/applicationsets/00-mongodb.yaml | 6 +++--- argocd/countly-hosted/applicationsets/01-clickhouse.yaml | 6 +++--- argocd/countly-hosted/applicationsets/02-kafka.yaml | 6 +++--- argocd/countly-hosted/applicationsets/03-countly.yaml | 6 +++--- argocd/countly-hosted/applicationsets/04-observability.yaml | 6 +++--- argocd/countly-hosted/applicationsets/05-migration.yaml | 6 +++--- argocd/countly-hosted/operators/00-cert-manager.yaml | 2 +- argocd/countly-hosted/operators/01-mongodb-crds.yaml | 2 +- argocd/countly-hosted/operators/02-mongodb-operator.yaml | 2 +- argocd/countly-hosted/operators/03-clickhouse-operator.yaml | 2 +- argocd/countly-hosted/operators/04-strimzi-operator.yaml | 2 +- argocd/countly-hosted/operators/05-nginx-ingress.yaml | 4 ++-- .../operators/06-letsencrypt-prod-issuer-app.yaml | 4 ++-- .../operators/07-external-secrets-operator.yaml | 2 +- .../countly-hosted/operators/08-cluster-secret-store.yaml | 4 ++-- argocd/countly-hosted/root-application.yaml | 2 +- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/argocd/countly-hosted/applicationsets/00-mongodb.yaml b/argocd/countly-hosted/applicationsets/00-mongodb.yaml index 8818e94..b694d78 100644 --- a/argocd/countly-hosted/applicationsets/00-mongodb.yaml +++ b/argocd/countly-hosted/applicationsets/00-mongodb.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: charts/countly-mongodb helm: releaseName: countly-mongodb @@ -40,7 +40,7 @@ spec: - name: global.security value: "{{ .security }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml index 6e384b0..3b6bc22 100644 --- a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml +++ b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: charts/countly-clickhouse helm: releaseName: countly-clickhouse @@ -40,7 +40,7 @@ spec: - name: global.security value: "{{ .security }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 1ad3ef2..51cec9b 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: charts/countly-kafka helm: releaseName: countly-kafka @@ -46,7 +46,7 @@ spec: - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/03-countly.yaml b/argocd/countly-hosted/applicationsets/03-countly.yaml index a46998e..1ec185e 100644 --- a/argocd/countly-hosted/applicationsets/03-countly.yaml +++ b/argocd/countly-hosted/applicationsets/03-countly.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: charts/countly helm: releaseName: countly @@ -52,7 +52,7 @@ spec: - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/04-observability.yaml b/argocd/countly-hosted/applicationsets/04-observability.yaml index 0b6f3ef..4dfa6c3 100644 --- a/argocd/countly-hosted/applicationsets/04-observability.yaml +++ b/argocd/countly-hosted/applicationsets/04-observability.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: '{{ if eq .observability "disabled" }}charts/noop{{ else }}charts/countly-observability{{ end }}' helm: releaseName: countly-observability @@ -43,7 +43,7 @@ spec: - name: global.observability value: "{{ .observability }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/05-migration.yaml b/argocd/countly-hosted/applicationsets/05-migration.yaml index de2f277..b98f485 100644 --- a/argocd/countly-hosted/applicationsets/05-migration.yaml +++ b/argocd/countly-hosted/applicationsets/05-migration.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: '{{ if eq .migration "enabled" }}charts/countly-migration{{ else }}charts/noop{{ end }}' helm: releaseName: countly-migration @@ -34,7 +34,7 @@ spec: - name: argocd.enabled value: "true" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: main + targetRevision: countly-hosted-argo-layout ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/operators/00-cert-manager.yaml b/argocd/countly-hosted/operators/00-cert-manager.yaml index 682eada..f64ad44 100644 --- a/argocd/countly-hosted/operators/00-cert-manager.yaml +++ b/argocd/countly-hosted/operators/00-cert-manager.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/01-mongodb-crds.yaml b/argocd/countly-hosted/operators/01-mongodb-crds.yaml index 3a90c1a..8bd883e 100644 --- a/argocd/countly-hosted/operators/01-mongodb-crds.yaml +++ b/argocd/countly-hosted/operators/01-mongodb-crds.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/02-mongodb-operator.yaml b/argocd/countly-hosted/operators/02-mongodb-operator.yaml index 89a7261..7dcbd0f 100644 --- a/argocd/countly-hosted/operators/02-mongodb-operator.yaml +++ b/argocd/countly-hosted/operators/02-mongodb-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml index d3a6212..a34e21e 100644 --- a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml +++ b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/04-strimzi-operator.yaml b/argocd/countly-hosted/operators/04-strimzi-operator.yaml index 51673cd..6891034 100644 --- a/argocd/countly-hosted/operators/04-strimzi-operator.yaml +++ b/argocd/countly-hosted/operators/04-strimzi-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/05-nginx-ingress.yaml b/argocd/countly-hosted/operators/05-nginx-ingress.yaml index 8f55a4a..34dcbcf 100644 --- a/argocd/countly-hosted/operators/05-nginx-ingress.yaml +++ b/argocd/countly-hosted/operators/05-nginx-ingress.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -29,7 +29,7 @@ spec: valueFiles: - $values/nginx-ingress-values.yaml - repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml index 2c1a78c..5e4ee33 100644 --- a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml +++ b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: '{{ if eq .tls "letsencrypt" }}argocd/operator-manifests/letsencrypt-prod-issuer{{ else }}charts/noop{{ end }}' directory: recurse: true diff --git a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml index 7d48bc5..533d101 100644 --- a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml +++ b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml index db32a30..ab7ea8a 100644 --- a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml +++ b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: main + revision: countly-hosted-argo-layout files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: charts/countly-cluster-secret-store helm: releaseName: countly-cluster-secret-store diff --git a/argocd/countly-hosted/root-application.yaml b/argocd/countly-hosted/root-application.yaml index ffce1cc..e5fd85a 100644 --- a/argocd/countly-hosted/root-application.yaml +++ b/argocd/countly-hosted/root-application.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: argocd/countly-hosted directory: recurse: true From 584999960867119153c2d4d734014def22826a5e Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 11:38:48 +0530 Subject: [PATCH 03/15] Split public and hosted Argo resource names --- argocd/applicationsets/00-mongodb.yaml | 2 +- argocd/applicationsets/01-clickhouse.yaml | 2 +- argocd/applicationsets/02-kafka.yaml | 2 +- argocd/applicationsets/03-countly.yaml | 2 +- argocd/applicationsets/04-observability.yaml | 2 +- argocd/applicationsets/05-migration.yaml | 2 +- argocd/countly-hosted/operators/00-cert-manager.yaml | 2 +- argocd/countly-hosted/operators/01-mongodb-crds.yaml | 2 +- argocd/countly-hosted/operators/02-mongodb-operator.yaml | 2 +- argocd/countly-hosted/operators/03-clickhouse-operator.yaml | 2 +- argocd/countly-hosted/operators/04-strimzi-operator.yaml | 2 +- argocd/countly-hosted/operators/05-nginx-ingress.yaml | 2 +- .../operators/06-letsencrypt-prod-issuer-app.yaml | 2 +- .../countly-hosted/operators/07-external-secrets-operator.yaml | 2 +- argocd/countly-hosted/operators/08-cluster-secret-store.yaml | 2 +- argocd/operators/00-cert-manager.yaml | 2 +- argocd/operators/01-mongodb-crds.yaml | 2 +- argocd/operators/02-mongodb-operator.yaml | 2 +- argocd/operators/03-clickhouse-operator.yaml | 2 +- argocd/operators/04-strimzi-operator.yaml | 2 +- argocd/operators/05-nginx-ingress.yaml | 2 +- argocd/operators/06-letsencrypt-prod-issuer-app.yaml | 2 +- argocd/operators/07-external-secrets-operator.yaml | 2 +- argocd/operators/08-cluster-secret-store.yaml | 2 +- argocd/projects/customers.yaml | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/argocd/applicationsets/00-mongodb.yaml b/argocd/applicationsets/00-mongodb.yaml index 4cbd4e3..c39d513 100644 --- a/argocd/applicationsets/00-mongodb.yaml +++ b/argocd/applicationsets/00-mongodb.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-mongodb + name: mongodb namespace: argocd spec: goTemplate: true diff --git a/argocd/applicationsets/01-clickhouse.yaml b/argocd/applicationsets/01-clickhouse.yaml index 817fe76..fef995b 100644 --- a/argocd/applicationsets/01-clickhouse.yaml +++ b/argocd/applicationsets/01-clickhouse.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-clickhouse + name: clickhouse namespace: argocd spec: goTemplate: true diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index db700d9..d9c2b4b 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-kafka + name: kafka namespace: argocd spec: goTemplate: true diff --git a/argocd/applicationsets/03-countly.yaml b/argocd/applicationsets/03-countly.yaml index 3e80361..5fa97cc 100644 --- a/argocd/applicationsets/03-countly.yaml +++ b/argocd/applicationsets/03-countly.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-app + name: app namespace: argocd spec: goTemplate: true diff --git a/argocd/applicationsets/04-observability.yaml b/argocd/applicationsets/04-observability.yaml index 1d69582..ba0b164 100644 --- a/argocd/applicationsets/04-observability.yaml +++ b/argocd/applicationsets/04-observability.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-observability + name: observability namespace: argocd spec: goTemplate: true diff --git a/argocd/applicationsets/05-migration.yaml b/argocd/applicationsets/05-migration.yaml index 5bf7143..1ed1523 100644 --- a/argocd/applicationsets/05-migration.yaml +++ b/argocd/applicationsets/05-migration.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: countly-migration + name: migration namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/00-cert-manager.yaml b/argocd/countly-hosted/operators/00-cert-manager.yaml index f64ad44..fb23812 100644 --- a/argocd/countly-hosted/operators/00-cert-manager.yaml +++ b/argocd/countly-hosted/operators/00-cert-manager.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-cert-manager + name: countly-cert-manager namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/01-mongodb-crds.yaml b/argocd/countly-hosted/operators/01-mongodb-crds.yaml index 8bd883e..b384810 100644 --- a/argocd/countly-hosted/operators/01-mongodb-crds.yaml +++ b/argocd/countly-hosted/operators/01-mongodb-crds.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-mongodb-crds + name: countly-mongodb-crds namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/02-mongodb-operator.yaml b/argocd/countly-hosted/operators/02-mongodb-operator.yaml index 7dcbd0f..c527a94 100644 --- a/argocd/countly-hosted/operators/02-mongodb-operator.yaml +++ b/argocd/countly-hosted/operators/02-mongodb-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-mongodb-kubernetes-operator + name: countly-mongodb-kubernetes-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml index a34e21e..0f7b9e3 100644 --- a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml +++ b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-clickhouse-operator + name: countly-clickhouse-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/04-strimzi-operator.yaml b/argocd/countly-hosted/operators/04-strimzi-operator.yaml index 6891034..be69198 100644 --- a/argocd/countly-hosted/operators/04-strimzi-operator.yaml +++ b/argocd/countly-hosted/operators/04-strimzi-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-strimzi-kafka-operator + name: countly-strimzi-kafka-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/05-nginx-ingress.yaml b/argocd/countly-hosted/operators/05-nginx-ingress.yaml index 34dcbcf..ad5afee 100644 --- a/argocd/countly-hosted/operators/05-nginx-ingress.yaml +++ b/argocd/countly-hosted/operators/05-nginx-ingress.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-nginx-ingress + name: countly-nginx-ingress namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml index 5e4ee33..74533f0 100644 --- a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml +++ b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-letsencrypt-prod-issuer + name: countly-letsencrypt-prod-issuer namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml index 533d101..5cfb75c 100644 --- a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml +++ b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-external-secrets + name: countly-external-secrets namespace: argocd spec: goTemplate: true diff --git a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml index ab7ea8a..0aa9a1c 100644 --- a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml +++ b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-cluster-secret-store + name: countly-cluster-secret-store namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/00-cert-manager.yaml b/argocd/operators/00-cert-manager.yaml index 4bd96cd..2e9c357 100644 --- a/argocd/operators/00-cert-manager.yaml +++ b/argocd/operators/00-cert-manager.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-cert-manager + name: cert-manager namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/01-mongodb-crds.yaml b/argocd/operators/01-mongodb-crds.yaml index ab09f9b..d28f67e 100644 --- a/argocd/operators/01-mongodb-crds.yaml +++ b/argocd/operators/01-mongodb-crds.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-mongodb-crds + name: mongodb-crds namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/02-mongodb-operator.yaml b/argocd/operators/02-mongodb-operator.yaml index 59a68df..db12424 100644 --- a/argocd/operators/02-mongodb-operator.yaml +++ b/argocd/operators/02-mongodb-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-mongodb-kubernetes-operator + name: mongodb-kubernetes-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/03-clickhouse-operator.yaml b/argocd/operators/03-clickhouse-operator.yaml index 569db87..68702a2 100644 --- a/argocd/operators/03-clickhouse-operator.yaml +++ b/argocd/operators/03-clickhouse-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-clickhouse-operator + name: clickhouse-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/04-strimzi-operator.yaml b/argocd/operators/04-strimzi-operator.yaml index 3d83405..902234b 100644 --- a/argocd/operators/04-strimzi-operator.yaml +++ b/argocd/operators/04-strimzi-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-strimzi-kafka-operator + name: strimzi-kafka-operator namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/05-nginx-ingress.yaml b/argocd/operators/05-nginx-ingress.yaml index 2883136..c6e23fb 100644 --- a/argocd/operators/05-nginx-ingress.yaml +++ b/argocd/operators/05-nginx-ingress.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-nginx-ingress + name: nginx-ingress namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/operators/06-letsencrypt-prod-issuer-app.yaml index 6046cf4..dabe1b6 100644 --- a/argocd/operators/06-letsencrypt-prod-issuer-app.yaml +++ b/argocd/operators/06-letsencrypt-prod-issuer-app.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-letsencrypt-prod-issuer + name: letsencrypt-prod-issuer namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/07-external-secrets-operator.yaml b/argocd/operators/07-external-secrets-operator.yaml index f20e732..180b494 100644 --- a/argocd/operators/07-external-secrets-operator.yaml +++ b/argocd/operators/07-external-secrets-operator.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-external-secrets + name: external-secrets namespace: argocd spec: goTemplate: true diff --git a/argocd/operators/08-cluster-secret-store.yaml b/argocd/operators/08-cluster-secret-store.yaml index 7a39007..5d9677d 100644 --- a/argocd/operators/08-cluster-secret-store.yaml +++ b/argocd/operators/08-cluster-secret-store.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: customer-cluster-secret-store + name: cluster-secret-store namespace: argocd spec: goTemplate: true diff --git a/argocd/projects/customers.yaml b/argocd/projects/customers.yaml index d48d4f7..5f6e275 100644 --- a/argocd/projects/customers.yaml +++ b/argocd/projects/customers.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: - name: countly-customers + name: customers namespace: argocd spec: description: "Shared AppProject for GitOps-managed Countly customer environments" From 8d8b3d6a244bfeba9d939069cf634873c2989845 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 11:42:12 +0530 Subject: [PATCH 04/15] Keep public bootstrap on test branch --- argocd/root-application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/root-application.yaml b/argocd/root-application.yaml index a84dae0..faa072c 100644 --- a/argocd/root-application.yaml +++ b/argocd/root-application.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: main + targetRevision: mutli-source-values-split path: argocd directory: recurse: true From c9ceeb8e5f770e0d98fc68faf05e0e72bf823342 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 11:48:53 +0530 Subject: [PATCH 05/15] Exclude hosted manifests from public bootstrap --- argocd/root-application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/root-application.yaml b/argocd/root-application.yaml index faa072c..25db1a5 100644 --- a/argocd/root-application.yaml +++ b/argocd/root-application.yaml @@ -11,7 +11,7 @@ spec: path: argocd directory: recurse: true - exclude: "{operator-manifests/**,customers/**}" + exclude: "{operator-manifests/**,customers/**,countly-hosted/**}" destination: server: https://kubernetes.default.svc namespace: argocd From c14bad1f3943c9a23cd1d0922d41249776d6e56c Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 13:34:31 +0530 Subject: [PATCH 06/15] Update Countly unified image digest --- charts/countly/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/countly/values.yaml b/charts/countly/values.yaml index d9efdaa..57fb282 100644 --- a/charts/countly/values.yaml +++ b/charts/countly/values.yaml @@ -51,7 +51,7 @@ image: repository: gcr.io/countly-dev-313620/countly-unified artifactRepository: countly-unified # -- Image digest (takes precedence over tag when set) - digest: "sha256:f81b39d4488c596f76a5c385d088a8998b7c1b20933366ad994f5315597ec48b" + digest: "sha256:b42efb9713ee11d173fe409924fb9e2a208b5c0beafed9e42f349b996b6650a4" # -- Image tag (used when digest is empty; defaults to appVersion) tag: "26.01" # -- Image pull policy From b7a1060b7585a88547d64759855a261cd3d59e33 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 14:11:17 +0530 Subject: [PATCH 07/15] Document Countly image override path --- environments/reference/countly.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environments/reference/countly.yaml b/environments/reference/countly.yaml index 5ae7cfe..4fc3856 100644 --- a/environments/reference/countly.yaml +++ b/environments/reference/countly.yaml @@ -27,6 +27,8 @@ serviceAccount: annotations: {} # --- Image --- +# Preferred place to pin or change the Countly app image for a deployment. +# Use digest for normal production changes; leave tag only as fallback. image: repository: gcr.io/countly-dev-313620/countly-unified artifactRepository: countly-unified From 7cddb3f7315c78a0579daf0658e097e4e09bdb62 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 14:15:07 +0530 Subject: [PATCH 08/15] Clarify Countly image source comments --- environments/reference/countly.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/reference/countly.yaml b/environments/reference/countly.yaml index 4fc3856..8198f06 100644 --- a/environments/reference/countly.yaml +++ b/environments/reference/countly.yaml @@ -30,8 +30,8 @@ serviceAccount: # Preferred place to pin or change the Countly app image for a deployment. # Use digest for normal production changes; leave tag only as fallback. image: - repository: gcr.io/countly-dev-313620/countly-unified - artifactRepository: countly-unified + repository: gcr.io/countly-dev-313620/countly-unified # Used when global.imageSource.mode=direct + artifactRepository: countly-unified # Used when global.imageSource.mode=gcpArtifactRegistry digest: "sha256:b42efb9713ee11d173fe409924fb9e2a208b5c0beafed9e42f349b996b6650a4" tag: "26.01" # Fallback when digest is empty pullPolicy: IfNotPresent From 0b504967c6f21957f61de4ff60c62a128a5b81d5 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 17:09:29 +0530 Subject: [PATCH 09/15] Support static nginx load balancer IPs --- argocd/README.md | 3 +++ argocd/countly-hosted/operators/05-nginx-ingress.yaml | 7 +++++++ argocd/operators/05-nginx-ingress.yaml | 7 +++++++ scripts/new-argocd-customer.sh | 3 +++ 4 files changed, 20 insertions(+) diff --git a/argocd/README.md b/argocd/README.md index af3b173..1123698 100644 --- a/argocd/README.md +++ b/argocd/README.md @@ -134,6 +134,9 @@ tls: letsencrypt observability: disabled kafkaConnect: balanced migration: disabled +nginxIngress: + service: + loadBalancerIP: "" # Optional: static public IP for the nginx LoadBalancer service ``` ### 3. Fill in the customer secrets diff --git a/argocd/countly-hosted/operators/05-nginx-ingress.yaml b/argocd/countly-hosted/operators/05-nginx-ingress.yaml index ad5afee..a513d55 100644 --- a/argocd/countly-hosted/operators/05-nginx-ingress.yaml +++ b/argocd/countly-hosted/operators/05-nginx-ingress.yaml @@ -28,6 +28,13 @@ spec: releaseName: nginx-ingress valueFiles: - $values/nginx-ingress-values.yaml + values: | + {{- $lbIP := dig "nginxIngress" "service" "loadBalancerIP" "" . -}} + {{- if $lbIP }} + controller: + service: + loadBalancerIP: {{ $lbIP | quote }} + {{- end }} - repoURL: https://github.com/Countly/helm.git targetRevision: mutli-source-values-split ref: values diff --git a/argocd/operators/05-nginx-ingress.yaml b/argocd/operators/05-nginx-ingress.yaml index c6e23fb..e6a0d0b 100644 --- a/argocd/operators/05-nginx-ingress.yaml +++ b/argocd/operators/05-nginx-ingress.yaml @@ -28,6 +28,13 @@ spec: releaseName: nginx-ingress valueFiles: - $values/nginx-ingress-values.yaml + values: | + {{- $lbIP := dig "nginxIngress" "service" "loadBalancerIP" "" . -}} + {{- if $lbIP }} + controller: + service: + loadBalancerIP: {{ $lbIP | quote }} + {{- end }} - repoURL: https://github.com/Countly/helm.git targetRevision: main ref: values diff --git a/scripts/new-argocd-customer.sh b/scripts/new-argocd-customer.sh index 2061c04..a9ea587 100755 --- a/scripts/new-argocd-customer.sh +++ b/scripts/new-argocd-customer.sh @@ -348,6 +348,9 @@ tls: letsencrypt observability: full kafkaConnect: balanced migration: disabled +nginxIngress: + service: + loadBalancerIP: "" # Optional: reserve a static GCP IP and set it here for the nginx LoadBalancer EOF cat < Date: Fri, 3 Apr 2026 20:50:43 +0530 Subject: [PATCH 10/15] Refactor Kafka Connect sizing overlays --- README.md | 11 +++++++---- argocd/applicationsets/02-kafka.yaml | 9 ++++++++- .../applicationsets/02-kafka.yaml | 9 ++++++++- charts/countly-argocd/templates/app-kafka.yaml | 5 ++++- charts/countly-argocd/values.schema.json | 3 ++- charts/countly-argocd/values.yaml | 1 + docs/ARGOCD.md | 3 ++- docs/DEPLOYING.md | 3 ++- docs/QUICKSTART.md | 3 ++- environments/example-production/global.yaml | 1 + environments/example-small/global.yaml | 1 + environments/local/global.yaml | 1 + environments/local/kafka.yaml | 3 ++- environments/reference/README.md | 3 ++- environments/reference/clickhouse.yaml | 2 +- environments/reference/countly.yaml | 2 +- environments/reference/global.yaml | 3 ++- environments/reference/kafka.yaml | 2 +- environments/reference/mongodb.yaml | 2 +- helmfile.yaml.gotmpl | 6 +++++- profiles/kafka-connect-sizing/local/kafka.yaml | 12 ++++++++++++ .../kafka-connect-sizing/production/kafka.yaml | 8 ++++++++ profiles/kafka-connect-sizing/small/kafka.yaml | 12 ++++++++++++ profiles/kafka-connect-sizing/tier1/kafka.yaml | 12 ++++++++++++ profiles/sizing/local/kafka.yaml | 13 ------------- profiles/sizing/production/kafka.yaml | 13 ------------- profiles/sizing/small/kafka.yaml | 13 ------------- profiles/sizing/tier1/kafka.yaml | 18 ------------------ 28 files changed, 98 insertions(+), 76 deletions(-) create mode 100644 profiles/kafka-connect-sizing/local/kafka.yaml create mode 100644 profiles/kafka-connect-sizing/production/kafka.yaml create mode 100644 profiles/kafka-connect-sizing/small/kafka.yaml create mode 100644 profiles/kafka-connect-sizing/tier1/kafka.yaml diff --git a/README.md b/README.md index 126936e..4ad0861 100644 --- a/README.md +++ b/README.md @@ -254,16 +254,17 @@ Operator and platform apps are pinned by Helm chart version in `argocd/operators ### Manual Installation (without Helmfile) Substitute your profile choices from `global.yaml` into the commands below. -The value file order must match the layering: global → sizing → dimension profiles → security → environment → secrets. +The value file order must match the layering: global -> Kafka Connect mode -> optional Kafka Connect sizing override -> sizing -> dimension profiles -> security -> environment -> secrets. ```bash # Shorthand — substitute these from your environments//global.yaml ENV=my-deployment -SIZING=local # local | small | production +SIZING=local # local | small | tier1 | production SECURITY=open # open | hardened TLS=selfSigned # none | selfSigned | letsencrypt | provided OBS=full # disabled | full | external-grafana | external KC=balanced # throughput | balanced | low-latency +KC_SIZING="" # optional: local | small | tier1 | production helm install countly-mongodb ./charts/countly-mongodb -n mongodb --create-namespace \ --wait --timeout 10m \ @@ -284,8 +285,9 @@ helm install countly-clickhouse ./charts/countly-clickhouse -n clickhouse --crea helm install countly-kafka ./charts/countly-kafka -n kafka --create-namespace \ --wait --timeout 10m \ -f environments/$ENV/global.yaml \ - -f profiles/sizing/$SIZING/kafka.yaml \ -f profiles/kafka-connect/$KC/kafka.yaml \ + ${KC_SIZING:+-f profiles/kafka-connect-sizing/$KC_SIZING/kafka.yaml} \ + -f profiles/sizing/$SIZING/kafka.yaml \ -f profiles/observability/$OBS/kafka.yaml \ -f profiles/security/$SECURITY/kafka.yaml \ -f environments/$ENV/kafka.yaml \ @@ -378,9 +380,10 @@ helm/ countly-migration/ countly-argocd/ profiles/ # Composable profile dimensions - sizing/ # local | small | production + sizing/ # local | small | tier1 | production observability/ # disabled | full | external-grafana | external kafka-connect/ # throughput | balanced | low-latency + kafka-connect-sizing/ # optional validated per-tier Kafka Connect overrides tls/ # none | letsencrypt | provided | selfSigned security/ # open | hardened environments/ # Deployment environments diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index d9c2b4b..83c7921 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -28,8 +28,11 @@ spec: releaseName: countly-kafka valueFiles: - "../../environments/{{ .environment }}/global.yaml" - - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" + {{- if .kafkaConnectSizing }} + - "../../profiles/kafka-connect-sizing/{{ .kafkaConnectSizing }}/kafka.yaml" + {{- end }} + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "../../environments/{{ .environment }}/kafka.yaml" @@ -45,6 +48,10 @@ spec: value: "{{ .observability }}" - name: global.kafkaConnect value: "{{ .kafkaConnect }}" + {{- if .kafkaConnectSizing }} + - name: global.kafkaConnectSizing + value: "{{ .kafkaConnectSizing }}" + {{- end }} destination: server: "{{ .server }}" namespace: kafka diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 51cec9b..aaa33d2 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -27,8 +27,11 @@ spec: helm: releaseName: countly-kafka valueFiles: - - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" + {{- if .kafkaConnectSizing }} + - "../../profiles/kafka-connect-sizing/{{ .kafkaConnectSizing }}/kafka.yaml" + {{- end }} + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "$values/environments/{{ .environment }}/global.yaml" @@ -45,6 +48,10 @@ spec: value: "{{ .observability }}" - name: global.kafkaConnect value: "{{ .kafkaConnect }}" + {{- if .kafkaConnectSizing }} + - name: global.kafkaConnectSizing + value: "{{ .kafkaConnectSizing }}" + {{- end }} - repoURL: https://github.com/Countly/countly-deployment.git targetRevision: countly-hosted-argo-layout ref: values diff --git a/charts/countly-argocd/templates/app-kafka.yaml b/charts/countly-argocd/templates/app-kafka.yaml index 9ad325f..0c09db1 100644 --- a/charts/countly-argocd/templates/app-kafka.yaml +++ b/charts/countly-argocd/templates/app-kafka.yaml @@ -20,8 +20,11 @@ spec: releaseName: countly-kafka valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - - ../../profiles/sizing/{{ .Values.global.sizing }}/kafka.yaml - ../../profiles/kafka-connect/{{ .Values.global.kafkaConnect }}/kafka.yaml + {{- if .Values.global.kafkaConnectSizing }} + - ../../profiles/kafka-connect-sizing/{{ .Values.global.kafkaConnectSizing }}/kafka.yaml + {{- end }} + - ../../profiles/sizing/{{ .Values.global.sizing }}/kafka.yaml - ../../profiles/observability/{{ .Values.global.observability }}/kafka.yaml - ../../profiles/security/{{ .Values.global.security }}/kafka.yaml - ../../environments/{{ .Values.environment }}/kafka.yaml diff --git a/charts/countly-argocd/values.schema.json b/charts/countly-argocd/values.schema.json index 03c0c44..b790775 100644 --- a/charts/countly-argocd/values.schema.json +++ b/charts/countly-argocd/values.schema.json @@ -40,7 +40,8 @@ "security": { "type": "string" }, "tls": { "type": "string" }, "observability": { "type": "string" }, - "kafkaConnect": { "type": "string" } + "kafkaConnect": { "type": "string" }, + "kafkaConnectSizing": { "type": "string" } } }, "mongodb": { diff --git a/charts/countly-argocd/values.yaml b/charts/countly-argocd/values.yaml index 489ead7..e2c53f2 100644 --- a/charts/countly-argocd/values.yaml +++ b/charts/countly-argocd/values.yaml @@ -20,6 +20,7 @@ global: tls: letsencrypt observability: full kafkaConnect: balanced + kafkaConnectSizing: production # -- Component toggles mongodb: diff --git a/docs/ARGOCD.md b/docs/ARGOCD.md index c96382a..7c12ac5 100644 --- a/docs/ARGOCD.md +++ b/docs/ARGOCD.md @@ -498,11 +498,12 @@ project: "" # Profile selections (passed to child charts via valueFiles) global: - sizing: production # local | small | production + sizing: production # local | small | tier1 | production security: hardened # open | hardened tls: letsencrypt # none | letsencrypt | provided | selfSigned observability: full # disabled | full | external-grafana | external kafkaConnect: balanced # throughput | balanced | low-latency + kafkaConnectSizing: "" # optional Kafka Connect tier override # Component toggles mongodb: diff --git a/docs/DEPLOYING.md b/docs/DEPLOYING.md index 9bd328b..142980a 100644 --- a/docs/DEPLOYING.md +++ b/docs/DEPLOYING.md @@ -23,9 +23,10 @@ Edit `environments/my-deployment/global.yaml`: ```yaml global: - sizing: production # Sizing: local | small | production + sizing: production # Sizing: local | small | tier1 | production observability: full # Observability: disabled | full | external-grafana | external kafkaConnect: balanced # Kafka Connect: throughput | balanced | low-latency + kafkaConnectSizing: "" # Optional Kafka Connect tier override: local | small | tier1 | production tls: letsencrypt # TLS: none | letsencrypt | provided | selfSigned security: hardened # Security: open | hardened storageClass: gp3 # Your cluster's storage class diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 2a8dedf..cfd3c25 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -113,8 +113,9 @@ helm install countly-kafka ./charts/countly-kafka \ -n kafka --create-namespace \ --wait --timeout 10m \ -f environments/local/global.yaml \ - -f profiles/sizing/local/kafka.yaml \ -f profiles/kafka-connect/balanced/kafka.yaml \ + -f profiles/kafka-connect-sizing/local/kafka.yaml \ + -f profiles/sizing/local/kafka.yaml \ -f profiles/observability/full/kafka.yaml \ -f profiles/security/open/kafka.yaml \ -f environments/local/kafka.yaml \ diff --git a/environments/example-production/global.yaml b/environments/example-production/global.yaml index 9ed5760..196328c 100644 --- a/environments/example-production/global.yaml +++ b/environments/example-production/global.yaml @@ -6,6 +6,7 @@ global: tls: letsencrypt observability: full kafkaConnect: balanced + kafkaConnectSizing: production imageSource: mode: gcpArtifactRegistry gcpArtifactRegistry: diff --git a/environments/example-small/global.yaml b/environments/example-small/global.yaml index 1726de3..c1896c2 100644 --- a/environments/example-small/global.yaml +++ b/environments/example-small/global.yaml @@ -6,6 +6,7 @@ global: tls: none observability: full kafkaConnect: balanced + kafkaConnectSizing: small ingress: hostname: analytics-dev.example.com diff --git a/environments/local/global.yaml b/environments/local/global.yaml index 1e83cc8..10ef049 100644 --- a/environments/local/global.yaml +++ b/environments/local/global.yaml @@ -4,3 +4,4 @@ global: tls: selfSigned observability: full kafkaConnect: balanced + kafkaConnectSizing: local diff --git a/environments/local/kafka.yaml b/environments/local/kafka.yaml index ed80d66..c31cef5 100644 --- a/environments/local/kafka.yaml +++ b/environments/local/kafka.yaml @@ -1,5 +1,6 @@ # Local environment — Kafka chart overrides (non-sizing) -# Profile defaults come from profiles/sizing/local/kafka.yaml +# Profile defaults come from profiles/sizing/local/kafka.yaml and +# profiles/kafka-connect-sizing/local/kafka.yaml # Credentials come from credentials-kafka.yaml # Use OTel-enabled image (includes /opt/otel/opentelemetry-javaagent.jar) diff --git a/environments/reference/README.md b/environments/reference/README.md index c83d349..00bc0aa 100644 --- a/environments/reference/README.md +++ b/environments/reference/README.md @@ -11,10 +11,11 @@ This directory is a complete starting point for a new Countly deployment. 2. Edit `global.yaml`: - Set `ingress.hostname` to your domain - - Choose `global.sizing`: `local`, `small`, or `production` + - Choose `global.sizing`: `local`, `small`, `tier1`, or `production` - Choose `global.tls`: `none`, `letsencrypt`, `provided`, or `selfSigned` - Choose `global.observability`: `disabled`, `full`, `external-grafana`, or `external` - Choose `global.kafkaConnect`: `throughput`, `balanced`, or `low-latency` + - Optionally set `global.kafkaConnectSizing` to `local`, `small`, `tier1`, or `production` when you need a validated Kafka Connect override for that hardware tier - Choose `global.security`: `open` or `hardened` - Choose backing service modes (bundled or external) - For GAR, set `global.imageSource`, `global.imagePullSecrets`, and optionally `global.imagePullSecretExternalSecret` diff --git a/environments/reference/clickhouse.yaml b/environments/reference/clickhouse.yaml index 22b2187..f860479 100644 --- a/environments/reference/clickhouse.yaml +++ b/environments/reference/clickhouse.yaml @@ -10,7 +10,7 @@ global: imageRegistry: "" imagePullSecrets: [] storageClass: "" - sizing: small # local | small | production + sizing: small # local | small | tier1 | production scheduling: nodeSelector: {} tolerations: [] diff --git a/environments/reference/countly.yaml b/environments/reference/countly.yaml index 8198f06..c9c82e0 100644 --- a/environments/reference/countly.yaml +++ b/environments/reference/countly.yaml @@ -10,7 +10,7 @@ global: imageRegistry: "" imagePullSecrets: [] storageClass: "" - sizing: small # local | small | production + sizing: small # local | small | tier1 | production scheduling: nodeSelector: {} tolerations: [] diff --git a/environments/reference/global.yaml b/environments/reference/global.yaml index a487420..9cba25b 100644 --- a/environments/reference/global.yaml +++ b/environments/reference/global.yaml @@ -10,9 +10,10 @@ global: # --- Profile Selectors --- - sizing: production # local | small | production + sizing: production # local | small | tier1 | production observability: full # disabled | full | external-grafana | external kafkaConnect: balanced # throughput | balanced | low-latency + kafkaConnectSizing: "" # optional: local | small | tier1 | production tls: letsencrypt # none | letsencrypt | provided | selfSigned security: open # open | hardened diff --git a/environments/reference/kafka.yaml b/environments/reference/kafka.yaml index 6100f36..1c3c4e3 100644 --- a/environments/reference/kafka.yaml +++ b/environments/reference/kafka.yaml @@ -14,7 +14,7 @@ global: repositoryPrefix: "" imagePullSecrets: [] storageClass: "" - sizing: small # local | small | production + sizing: small # local | small | tier1 | production scheduling: nodeSelector: {} tolerations: [] diff --git a/environments/reference/mongodb.yaml b/environments/reference/mongodb.yaml index 5631bd2..0c8d254 100644 --- a/environments/reference/mongodb.yaml +++ b/environments/reference/mongodb.yaml @@ -10,7 +10,7 @@ global: imageRegistry: "" imagePullSecrets: [] storageClass: "" - sizing: small # local | small | production + sizing: small # local | small | tier1 | production scheduling: nodeSelector: {} tolerations: [] diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index 90b78fe..cebd3a2 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -55,8 +55,12 @@ releases: namespace: kafka values: - environments/{{ .Environment.Name }}/global.yaml - - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/kafka.yaml - profiles/kafka-connect/{{ .Values | get "global.kafkaConnect" "balanced" }}/kafka.yaml + {{- $kcSizing := .Values | get "global.kafkaConnectSizing" "" }} + {{- if ne $kcSizing "" }} + - profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml + {{- end }} + - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/kafka.yaml - profiles/observability/{{ .Values | get "global.observability" "full" }}/kafka.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/kafka.yaml - environments/{{ .Environment.Name }}/kafka.yaml diff --git a/profiles/kafka-connect-sizing/local/kafka.yaml b/profiles/kafka-connect-sizing/local/kafka.yaml new file mode 100644 index 0000000..4c09932 --- /dev/null +++ b/profiles/kafka-connect-sizing/local/kafka.yaml @@ -0,0 +1,12 @@ +kafkaConnect: + replicas: 1 + resources: + requests: { cpu: "500m", memory: "2Gi" } + limits: { cpu: "1", memory: "2Gi" } + jvmOptions: + xms: "1g" + xmx: "1g" + workerConfig: + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 diff --git a/profiles/kafka-connect-sizing/production/kafka.yaml b/profiles/kafka-connect-sizing/production/kafka.yaml new file mode 100644 index 0000000..895b608 --- /dev/null +++ b/profiles/kafka-connect-sizing/production/kafka.yaml @@ -0,0 +1,8 @@ +kafkaConnect: + replicas: 2 + resources: + requests: { cpu: "2", memory: "8Gi" } + limits: { cpu: "2", memory: "8Gi" } + jvmOptions: + xms: "5g" + xmx: "5g" diff --git a/profiles/kafka-connect-sizing/small/kafka.yaml b/profiles/kafka-connect-sizing/small/kafka.yaml new file mode 100644 index 0000000..4c09932 --- /dev/null +++ b/profiles/kafka-connect-sizing/small/kafka.yaml @@ -0,0 +1,12 @@ +kafkaConnect: + replicas: 1 + resources: + requests: { cpu: "500m", memory: "2Gi" } + limits: { cpu: "1", memory: "2Gi" } + jvmOptions: + xms: "1g" + xmx: "1g" + workerConfig: + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 diff --git a/profiles/kafka-connect-sizing/tier1/kafka.yaml b/profiles/kafka-connect-sizing/tier1/kafka.yaml new file mode 100644 index 0000000..a5699cc --- /dev/null +++ b/profiles/kafka-connect-sizing/tier1/kafka.yaml @@ -0,0 +1,12 @@ +kafkaConnect: + replicas: 1 + resources: + requests: { cpu: "1", memory: "2Gi" } + limits: { cpu: "1", memory: "2Gi" } + jvmOptions: + xms: "1g" + xmx: "1g" + workerConfig: + config.storage.replication.factor: 2 + offset.storage.replication.factor: 2 + status.storage.replication.factor: 2 diff --git a/profiles/sizing/local/kafka.yaml b/profiles/sizing/local/kafka.yaml index 601b3bc..05785db 100644 --- a/profiles/sizing/local/kafka.yaml +++ b/profiles/sizing/local/kafka.yaml @@ -28,16 +28,3 @@ controllers: cruiseControl: enabled: false - -kafkaConnect: - replicas: 1 - resources: - requests: { cpu: "500m", memory: "2Gi" } - limits: { cpu: "1", memory: "2Gi" } - jvmOptions: - xms: "1g" - xmx: "1g" - workerConfig: - config.storage.replication.factor: 1 - offset.storage.replication.factor: 1 - status.storage.replication.factor: 1 diff --git a/profiles/sizing/production/kafka.yaml b/profiles/sizing/production/kafka.yaml index 60af3cd..9b6b78b 100644 --- a/profiles/sizing/production/kafka.yaml +++ b/profiles/sizing/production/kafka.yaml @@ -6,10 +6,6 @@ # brokers.config.offsets.topic.replication.factor (default 3) # brokers.config.transaction.state.log.replication.factor (default 3) # brokers.config.transaction.state.log.min.isr (default 2) -# kafkaConnect.workerConfig.config.storage.replication.factor (default 2) -# kafkaConnect.workerConfig.offset.storage.replication.factor (default 2) -# kafkaConnect.workerConfig.status.storage.replication.factor (default 2) - brokers: replicas: 3 resources: @@ -51,12 +47,3 @@ cruiseControl: resources: requests: { cpu: "1", memory: "2Gi" } limits: { cpu: "1", memory: "2Gi" } - -kafkaConnect: - replicas: 2 - resources: - requests: { cpu: "2", memory: "8Gi" } - limits: { cpu: "2", memory: "8Gi" } - jvmOptions: - xms: "5g" - xmx: "5g" diff --git a/profiles/sizing/small/kafka.yaml b/profiles/sizing/small/kafka.yaml index faec228..2d39ee5 100644 --- a/profiles/sizing/small/kafka.yaml +++ b/profiles/sizing/small/kafka.yaml @@ -28,16 +28,3 @@ controllers: cruiseControl: enabled: false - -kafkaConnect: - replicas: 1 - resources: - requests: { cpu: "500m", memory: "2Gi" } - limits: { cpu: "1", memory: "2Gi" } - jvmOptions: - xms: "1g" - xmx: "1g" - workerConfig: - config.storage.replication.factor: 1 - offset.storage.replication.factor: 1 - status.storage.replication.factor: 1 diff --git a/profiles/sizing/tier1/kafka.yaml b/profiles/sizing/tier1/kafka.yaml index 35aa866..28e687b 100644 --- a/profiles/sizing/tier1/kafka.yaml +++ b/profiles/sizing/tier1/kafka.yaml @@ -4,11 +4,6 @@ # 2 brokers + 1 controller. Replication factor 2, min.insync.replicas 1 # (allows writes when one broker is unavailable on a 2-broker cluster). # -# NOTE: kafkaConnect resources here will be overridden by the kafka-connect -# dimension profile (throughput/balanced/low-latency). The values below -# reflect the tier1-validated connect worker sizing; pair with a matching -# kafka-connect profile if you need to honour them exactly. - brokers: replicas: 2 resources: @@ -38,16 +33,3 @@ controllers: cruiseControl: enabled: false - -kafkaConnect: - replicas: 1 - resources: - requests: { cpu: "1", memory: "2Gi" } - limits: { cpu: "1", memory: "2Gi" } - jvmOptions: - xms: "1g" - xmx: "1g" - workerConfig: - config.storage.replication.factor: 2 - offset.storage.replication.factor: 2 - status.storage.replication.factor: 2 From 1246f7f5ba1221779c8eb732d1696dbf552128ff Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 22:31:11 +0530 Subject: [PATCH 11/15] Make sizing profiles win over env defaults --- argocd/ONBOARDING.md | 289 ++++++++++++++++++ argocd/README.md | 1 + argocd/applicationsets/00-mongodb.yaml | 2 +- argocd/applicationsets/01-clickhouse.yaml | 2 +- argocd/applicationsets/02-kafka.yaml | 12 +- argocd/applicationsets/03-countly.yaml | 2 +- argocd/applicationsets/04-observability.yaml | 2 +- .../applicationsets/00-mongodb.yaml | 2 +- .../applicationsets/01-clickhouse.yaml | 2 +- .../applicationsets/02-kafka.yaml | 12 +- .../applicationsets/03-countly.yaml | 2 +- .../applicationsets/04-observability.yaml | 2 +- .../templates/app-clickhouse.yaml | 2 +- .../countly-argocd/templates/app-countly.yaml | 2 +- .../countly-argocd/templates/app-kafka.yaml | 12 +- .../countly-argocd/templates/app-mongodb.yaml | 2 +- .../templates/app-observability.yaml | 2 +- charts/countly-argocd/values.yaml | 2 +- environments/reference/global.yaml | 2 +- helmfile.yaml.gotmpl | 19 +- scripts/new-argocd-customer.sh | 3 + 21 files changed, 342 insertions(+), 34 deletions(-) diff --git a/argocd/ONBOARDING.md b/argocd/ONBOARDING.md index 33e0fa3..614450c 100644 --- a/argocd/ONBOARDING.md +++ b/argocd/ONBOARDING.md @@ -274,6 +274,7 @@ security: hardened tls: letsencrypt observability: disabled kafkaConnect: balanced +kafkaConnectSizing: auto migration: disabled ``` @@ -619,6 +620,294 @@ Healthy looks like: If you see `InvalidProviderConfig`, first check Workload Identity. +## Production Identity Model + +This is the recommended production setup when you manage many customer clusters. + +Use two identities: + +1. One shared Argo deploy identity + - used only to deploy Kubernetes resources into customer clusters + - shared across customers is fine + - this is your platform control-plane identity + +2. One separate runtime Google service account per customer cluster + - used by workloads inside that customer cluster + - used for: + - pulling images from Artifact Registry + - reading secrets from Secret Manager + - this should not have cluster-admin rights + +### Sharing Guide + +| Case | Recommendation | +|------|----------------| +| One shared Argo deploy identity for all customers | Shared allowed | +| One shared runtime identity for all customers for image pulls only | Shared acceptable with caution | +| One shared runtime identity for all customers for Secret Manager access | Should be separate | +| One shared identity for deploy + runtime + secrets | Should be separate | + +### Permissions Matrix + +| Identity | Scope | Recommended access | +|----------|-------|--------------------| +| Argo deploy identity | Shared/platform | Kubernetes deploy access to target clusters only | +| Runtime customer identity | Per customer/cluster | `roles/artifactregistry.reader`, `roles/secretmanager.secretAccessor` | +| Optional image-pull-only identity | Per customer or shared | `roles/artifactregistry.reader` only | + +### Step-By-Step Production Setup + +#### 1. Create The Customer Cluster + +Who runs this: +- platform or infrastructure engineer + +Example: + +```bash +gcloud container clusters create CUSTOMER_CLUSTER \ + --project=PROJECT_ID \ + --zone=ZONE \ + --workload-pool=PROJECT_ID.svc.id.goog +``` + +If the cluster already exists, verify Workload Identity: + +```bash +gcloud container clusters describe CUSTOMER_CLUSTER \ + --project=PROJECT_ID \ + --zone=ZONE \ + --format="value(workloadIdentityConfig.workloadPool)" +``` + +Healthy output: + +```text +PROJECT_ID.svc.id.goog +``` + +#### 2. Ensure The Node Pool Uses GKE Metadata + +Who runs this: +- platform or infrastructure engineer + +Check: + +```bash +gcloud container node-pools describe default-pool \ + --cluster=CUSTOMER_CLUSTER \ + --project=PROJECT_ID \ + --zone=ZONE \ + --format="value(config.workloadMetadataConfig.mode)" +``` + +If needed: + +```bash +gcloud container node-pools update default-pool \ + --cluster=CUSTOMER_CLUSTER \ + --project=PROJECT_ID \ + --zone=ZONE \ + --workload-metadata=GKE_METADATA +``` + +Why this matters: +- GKE Standard needs this for Workload Identity to function correctly + +#### 3. Create The Per-Customer Runtime Google Service Account + +Who runs this: +- platform or infrastructure engineer + +Example: + +```bash +gcloud iam service-accounts create CUSTOMER-runtime \ + --project=PROJECT_ID \ + --display-name="CUSTOMER runtime identity" +``` + +This creates: + +```text +CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com +``` + +#### 4. Grant Runtime Cloud Permissions + +Who runs this: +- platform or infrastructure engineer + +Artifact Registry read: + +```bash +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member="serviceAccount:CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/artifactregistry.reader" +``` + +Secret Manager read, simple project-wide version: + +```bash +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member="serviceAccount:CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +Better least-privilege version, grant only on specific secrets: + +```bash +gcloud secrets add-iam-policy-binding SECRET_NAME \ + --project=PROJECT_ID \ + --member="serviceAccount:CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +#### 5. Bind The Kubernetes Service Account To The Google Service Account + +Who runs this: +- platform engineer or GitOps owner + +This is the Workload Identity link. + +Grant impersonation: + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com \ + --project=PROJECT_ID \ + --role="roles/iam.workloadIdentityUser" \ + --member="serviceAccount:PROJECT_ID.svc.id.goog[NAMESPACE/KSA_NAME]" +``` + +Typical examples: +- `external-secrets/external-secrets` +- `countly/countly` + +Annotate the Kubernetes service account: + +```bash +kubectl annotate serviceaccount KSA_NAME \ + -n NAMESPACE \ + iam.gke.io/gcp-service-account=CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com \ + --overwrite +``` + +#### 6. Create Customer Secrets In Secret Manager + +Who runs this: +- platform engineer or secrets owner + +Example: + +```bash +gcloud secrets create CUSTOMER-mongodb-app-password \ + --project=PROJECT_ID \ + --replication-policy=user-managed \ + --locations=us-central1 + +printf '%s' 'StrongPasswordHere' | \ +gcloud secrets versions add CUSTOMER-mongodb-app-password \ + --project=PROJECT_ID \ + --data-file=- +``` + +Repeat for your customer-specific application secrets. + +If you use shared TLS for many customers, create these once: + +```text +countly-prod-tls-crt +countly-prod-tls-key +``` + +#### 7. Add The Cluster To Argo CD + +Who runs this: +- GitOps or platform engineer + +Get kube credentials: + +```bash +gcloud container clusters get-credentials CUSTOMER_CLUSTER \ + --project=PROJECT_ID \ + --zone=ZONE +``` + +Add cluster to Argo: + +```bash +argocd cluster add CURRENT_KUBE_CONTEXT +``` + +Check: + +```bash +argocd cluster list +``` + +Important: +- this gives Argo Kubernetes access to deploy resources +- this is separate from the runtime Google service account + +#### 8. Create The Customer Overlay In Git + +Who runs this: +- GitOps or platform engineer + +For Secret Manager mode: + +```bash +./scripts/new-argocd-customer.sh --secret-mode gcp-secrets CUSTOMER https://CLUSTER_ENDPOINT CUSTOMER.example.com +``` + +Then fill: +- `argocd/customers/CUSTOMER.yaml` +- `environments/CUSTOMER/global.yaml` + +Typical values: +- `gcpServiceAccountEmail: CUSTOMER-runtime@PROJECT_ID.iam.gserviceaccount.com` +- `secretManagerProjectID: PROJECT_ID` +- `clusterProjectID: PROJECT_ID` +- `clusterName: CUSTOMER_CLUSTER` +- `clusterLocation: ZONE` + +#### 9. Commit And Sync + +Who runs this: +- GitOps or platform engineer + +```bash +git add argocd/customers/CUSTOMER.yaml environments/CUSTOMER +git commit -m "Add CUSTOMER customer" +git push origin BRANCH +``` + +Then: + +```bash +argocd app get countly-bootstrap --hard-refresh +argocd app sync countly-bootstrap +``` + +#### 10. Verify + +Who runs this: +- GitOps or platform engineer + +```bash +kubectl get applications -n argocd +kubectl get externalsecrets.external-secrets.io -A +kubectl get pods -A +kubectl get ingress -n countly +``` + +### What Not To Do + +- do not use service account keys for workloads if Workload Identity is available +- do not give the runtime service account cluster-admin +- do not use one broad Secret Manager runtime identity for every customer if you can avoid it + ## Step 6: Create Secrets In Google Secret Manager Use names like: diff --git a/argocd/README.md b/argocd/README.md index 1123698..e8b3cfc 100644 --- a/argocd/README.md +++ b/argocd/README.md @@ -133,6 +133,7 @@ security: open tls: letsencrypt observability: disabled kafkaConnect: balanced +kafkaConnectSizing: auto migration: disabled nginxIngress: service: diff --git a/argocd/applicationsets/00-mongodb.yaml b/argocd/applicationsets/00-mongodb.yaml index c39d513..be9adcd 100644 --- a/argocd/applicationsets/00-mongodb.yaml +++ b/argocd/applicationsets/00-mongodb.yaml @@ -28,9 +28,9 @@ spec: releaseName: countly-mongodb valueFiles: - "../../environments/{{ .environment }}/global.yaml" - - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" - "../../profiles/security/{{ .security }}/mongodb.yaml" - "../../environments/{{ .environment }}/mongodb.yaml" + - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" - "../../environments/{{ .environment }}/credentials-mongodb.yaml" parameters: - name: argocd.enabled diff --git a/argocd/applicationsets/01-clickhouse.yaml b/argocd/applicationsets/01-clickhouse.yaml index fef995b..ea28c26 100644 --- a/argocd/applicationsets/01-clickhouse.yaml +++ b/argocd/applicationsets/01-clickhouse.yaml @@ -28,9 +28,9 @@ spec: releaseName: countly-clickhouse valueFiles: - "../../environments/{{ .environment }}/global.yaml" - - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" - "../../profiles/security/{{ .security }}/clickhouse.yaml" - "../../environments/{{ .environment }}/clickhouse.yaml" + - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" - "../../environments/{{ .environment }}/credentials-clickhouse.yaml" parameters: - name: argocd.enabled diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index 83c7921..fd05f9d 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -29,13 +29,17 @@ spec: valueFiles: - "../../environments/{{ .environment }}/global.yaml" - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" - {{- if .kafkaConnectSizing }} - - "../../profiles/kafka-connect-sizing/{{ .kafkaConnectSizing }}/kafka.yaml" - {{- end }} - - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "../../environments/{{ .environment }}/kafka.yaml" + {{- $kcSizing := .kafkaConnectSizing | default "auto" }} + {{- if eq $kcSizing "auto" }} + {{- $kcSizing = .sizing }} + {{- end }} + {{- if $kcSizing }} + - "../../profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml" + {{- end }} + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../environments/{{ .environment }}/credentials-kafka.yaml" parameters: - name: argocd.enabled diff --git a/argocd/applicationsets/03-countly.yaml b/argocd/applicationsets/03-countly.yaml index 5fa97cc..23ba50c 100644 --- a/argocd/applicationsets/03-countly.yaml +++ b/argocd/applicationsets/03-countly.yaml @@ -28,11 +28,11 @@ spec: releaseName: countly valueFiles: - "../../environments/{{ .environment }}/global.yaml" - - "../../profiles/sizing/{{ .sizing }}/countly.yaml" - "../../profiles/tls/{{ .tls }}/countly.yaml" - "../../profiles/observability/{{ .observability }}/countly.yaml" - "../../profiles/security/{{ .security }}/countly.yaml" - "../../environments/{{ .environment }}/countly.yaml" + - "../../profiles/sizing/{{ .sizing }}/countly.yaml" - "../../environments/{{ .environment }}/credentials-countly.yaml" parameters: - name: argocd.enabled diff --git a/argocd/applicationsets/04-observability.yaml b/argocd/applicationsets/04-observability.yaml index ba0b164..94a372b 100644 --- a/argocd/applicationsets/04-observability.yaml +++ b/argocd/applicationsets/04-observability.yaml @@ -28,10 +28,10 @@ spec: releaseName: countly-observability valueFiles: - "../../environments/{{ .environment }}/global.yaml" - - "../../profiles/sizing/{{ .sizing }}/observability.yaml" - "../../profiles/observability/{{ .observability }}/observability.yaml" - "../../profiles/security/{{ .security }}/observability.yaml" - "../../environments/{{ .environment }}/observability.yaml" + - "../../profiles/sizing/{{ .sizing }}/observability.yaml" - "../../environments/{{ .environment }}/credentials-observability.yaml" parameters: - name: argocd.enabled diff --git a/argocd/countly-hosted/applicationsets/00-mongodb.yaml b/argocd/countly-hosted/applicationsets/00-mongodb.yaml index b694d78..f706932 100644 --- a/argocd/countly-hosted/applicationsets/00-mongodb.yaml +++ b/argocd/countly-hosted/applicationsets/00-mongodb.yaml @@ -27,10 +27,10 @@ spec: helm: releaseName: countly-mongodb valueFiles: - - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" - "../../profiles/security/{{ .security }}/mongodb.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/mongodb.yaml" + - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" - "$values/environments/{{ .environment }}/credentials-mongodb.yaml" parameters: - name: argocd.enabled diff --git a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml index 3b6bc22..ec567ed 100644 --- a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml +++ b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml @@ -27,10 +27,10 @@ spec: helm: releaseName: countly-clickhouse valueFiles: - - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" - "../../profiles/security/{{ .security }}/clickhouse.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/clickhouse.yaml" + - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" - "$values/environments/{{ .environment }}/credentials-clickhouse.yaml" parameters: - name: argocd.enabled diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index aaa33d2..574ad09 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -28,14 +28,18 @@ spec: releaseName: countly-kafka valueFiles: - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" - {{- if .kafkaConnectSizing }} - - "../../profiles/kafka-connect-sizing/{{ .kafkaConnectSizing }}/kafka.yaml" - {{- end }} - - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/kafka.yaml" + {{- $kcSizing := .kafkaConnectSizing | default "auto" }} + {{- if eq $kcSizing "auto" }} + {{- $kcSizing = .sizing }} + {{- end }} + {{- if $kcSizing }} + - "../../profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml" + {{- end }} + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "$values/environments/{{ .environment }}/credentials-kafka.yaml" parameters: - name: argocd.enabled diff --git a/argocd/countly-hosted/applicationsets/03-countly.yaml b/argocd/countly-hosted/applicationsets/03-countly.yaml index 1ec185e..0c0def9 100644 --- a/argocd/countly-hosted/applicationsets/03-countly.yaml +++ b/argocd/countly-hosted/applicationsets/03-countly.yaml @@ -27,12 +27,12 @@ spec: helm: releaseName: countly valueFiles: - - "../../profiles/sizing/{{ .sizing }}/countly.yaml" - "../../profiles/tls/{{ .tls }}/countly.yaml" - "../../profiles/observability/{{ .observability }}/countly.yaml" - "../../profiles/security/{{ .security }}/countly.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/countly.yaml" + - "../../profiles/sizing/{{ .sizing }}/countly.yaml" - "$values/environments/{{ .environment }}/credentials-countly.yaml" parameters: - name: argocd.enabled diff --git a/argocd/countly-hosted/applicationsets/04-observability.yaml b/argocd/countly-hosted/applicationsets/04-observability.yaml index 4dfa6c3..41a0da9 100644 --- a/argocd/countly-hosted/applicationsets/04-observability.yaml +++ b/argocd/countly-hosted/applicationsets/04-observability.yaml @@ -27,11 +27,11 @@ spec: helm: releaseName: countly-observability valueFiles: - - "../../profiles/sizing/{{ .sizing }}/observability.yaml" - "../../profiles/observability/{{ .observability }}/observability.yaml" - "../../profiles/security/{{ .security }}/observability.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/observability.yaml" + - "../../profiles/sizing/{{ .sizing }}/observability.yaml" - "$values/environments/{{ .environment }}/credentials-observability.yaml" parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/templates/app-clickhouse.yaml b/charts/countly-argocd/templates/app-clickhouse.yaml index fde892f..e5f8063 100644 --- a/charts/countly-argocd/templates/app-clickhouse.yaml +++ b/charts/countly-argocd/templates/app-clickhouse.yaml @@ -20,9 +20,9 @@ spec: releaseName: countly-clickhouse valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - - ../../profiles/sizing/{{ .Values.global.sizing }}/clickhouse.yaml - ../../profiles/security/{{ .Values.global.security }}/clickhouse.yaml - ../../environments/{{ .Values.environment }}/clickhouse.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/clickhouse.yaml - ../../environments/{{ .Values.environment }}/credentials-clickhouse.yaml parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/templates/app-countly.yaml b/charts/countly-argocd/templates/app-countly.yaml index e71346d..8698526 100644 --- a/charts/countly-argocd/templates/app-countly.yaml +++ b/charts/countly-argocd/templates/app-countly.yaml @@ -20,11 +20,11 @@ spec: releaseName: countly valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - - ../../profiles/sizing/{{ .Values.global.sizing }}/countly.yaml - ../../profiles/tls/{{ .Values.global.tls }}/countly.yaml - ../../profiles/observability/{{ .Values.global.observability }}/countly.yaml - ../../profiles/security/{{ .Values.global.security }}/countly.yaml - ../../environments/{{ .Values.environment }}/countly.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/countly.yaml - ../../environments/{{ .Values.environment }}/credentials-countly.yaml parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/templates/app-kafka.yaml b/charts/countly-argocd/templates/app-kafka.yaml index 0c09db1..e566c7b 100644 --- a/charts/countly-argocd/templates/app-kafka.yaml +++ b/charts/countly-argocd/templates/app-kafka.yaml @@ -1,4 +1,8 @@ {{- if .Values.kafka.enabled }} +{{- $kcSizing := default "auto" .Values.global.kafkaConnectSizing }} +{{- if eq $kcSizing "auto" }} +{{- $kcSizing = .Values.global.sizing }} +{{- end }} apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -21,13 +25,13 @@ spec: valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - ../../profiles/kafka-connect/{{ .Values.global.kafkaConnect }}/kafka.yaml - {{- if .Values.global.kafkaConnectSizing }} - - ../../profiles/kafka-connect-sizing/{{ .Values.global.kafkaConnectSizing }}/kafka.yaml - {{- end }} - - ../../profiles/sizing/{{ .Values.global.sizing }}/kafka.yaml - ../../profiles/observability/{{ .Values.global.observability }}/kafka.yaml - ../../profiles/security/{{ .Values.global.security }}/kafka.yaml - ../../environments/{{ .Values.environment }}/kafka.yaml + {{- if $kcSizing }} + - ../../profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml + {{- end }} + - ../../profiles/sizing/{{ .Values.global.sizing }}/kafka.yaml - ../../environments/{{ .Values.environment }}/credentials-kafka.yaml parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/templates/app-mongodb.yaml b/charts/countly-argocd/templates/app-mongodb.yaml index 86460b9..7649fb5 100644 --- a/charts/countly-argocd/templates/app-mongodb.yaml +++ b/charts/countly-argocd/templates/app-mongodb.yaml @@ -20,9 +20,9 @@ spec: releaseName: countly-mongodb valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - - ../../profiles/sizing/{{ .Values.global.sizing }}/mongodb.yaml - ../../profiles/security/{{ .Values.global.security }}/mongodb.yaml - ../../environments/{{ .Values.environment }}/mongodb.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/mongodb.yaml - ../../environments/{{ .Values.environment }}/credentials-mongodb.yaml parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/templates/app-observability.yaml b/charts/countly-argocd/templates/app-observability.yaml index 9876d82..e9e4615 100644 --- a/charts/countly-argocd/templates/app-observability.yaml +++ b/charts/countly-argocd/templates/app-observability.yaml @@ -20,10 +20,10 @@ spec: releaseName: countly-observability valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - - ../../profiles/sizing/{{ .Values.global.sizing }}/observability.yaml - ../../profiles/observability/{{ .Values.global.observability }}/observability.yaml - ../../profiles/security/{{ .Values.global.security }}/observability.yaml - ../../environments/{{ .Values.environment }}/observability.yaml + - ../../profiles/sizing/{{ .Values.global.sizing }}/observability.yaml - ../../environments/{{ .Values.environment }}/credentials-observability.yaml parameters: - name: argocd.enabled diff --git a/charts/countly-argocd/values.yaml b/charts/countly-argocd/values.yaml index e2c53f2..7ddbdb0 100644 --- a/charts/countly-argocd/values.yaml +++ b/charts/countly-argocd/values.yaml @@ -20,7 +20,7 @@ global: tls: letsencrypt observability: full kafkaConnect: balanced - kafkaConnectSizing: production + kafkaConnectSizing: auto # -- Component toggles mongodb: diff --git a/environments/reference/global.yaml b/environments/reference/global.yaml index 9cba25b..b8148c7 100644 --- a/environments/reference/global.yaml +++ b/environments/reference/global.yaml @@ -13,7 +13,7 @@ global: sizing: production # local | small | tier1 | production observability: full # disabled | full | external-grafana | external kafkaConnect: balanced # throughput | balanced | low-latency - kafkaConnectSizing: "" # optional: local | small | tier1 | production + kafkaConnectSizing: auto # auto uses global.sizing; set "" to disable, or choose local | small | tier1 | production tls: letsencrypt # none | letsencrypt | provided | selfSigned security: open # open | hardened diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index cebd3a2..6d185e6 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -33,9 +33,9 @@ releases: namespace: mongodb values: - environments/{{ .Environment.Name }}/global.yaml - - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/mongodb.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/mongodb.yaml - environments/{{ .Environment.Name }}/mongodb.yaml + - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/mongodb.yaml - environments/{{ .Environment.Name }}/credentials-mongodb.yaml - name: countly-clickhouse @@ -44,9 +44,9 @@ releases: namespace: clickhouse values: - environments/{{ .Environment.Name }}/global.yaml - - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/clickhouse.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/clickhouse.yaml - environments/{{ .Environment.Name }}/clickhouse.yaml + - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/clickhouse.yaml - environments/{{ .Environment.Name }}/credentials-clickhouse.yaml - name: countly-kafka @@ -56,14 +56,17 @@ releases: values: - environments/{{ .Environment.Name }}/global.yaml - profiles/kafka-connect/{{ .Values | get "global.kafkaConnect" "balanced" }}/kafka.yaml - {{- $kcSizing := .Values | get "global.kafkaConnectSizing" "" }} + - profiles/observability/{{ .Values | get "global.observability" "full" }}/kafka.yaml + - profiles/security/{{ .Values | get "global.security" "open" }}/kafka.yaml + - environments/{{ .Environment.Name }}/kafka.yaml + {{- $kcSizing := .Values | get "global.kafkaConnectSizing" "auto" }} + {{- if eq $kcSizing "auto" }} + {{- $kcSizing = .Values | get "global.sizing" "" }} + {{- end }} {{- if ne $kcSizing "" }} - profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml {{- end }} - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/kafka.yaml - - profiles/observability/{{ .Values | get "global.observability" "full" }}/kafka.yaml - - profiles/security/{{ .Values | get "global.security" "open" }}/kafka.yaml - - environments/{{ .Environment.Name }}/kafka.yaml - environments/{{ .Environment.Name }}/credentials-kafka.yaml needs: - mongodb/countly-mongodb @@ -74,11 +77,11 @@ releases: namespace: countly values: - environments/{{ .Environment.Name }}/global.yaml - - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/countly.yaml - profiles/tls/{{ .Values | get "global.tls" "none" }}/countly.yaml - profiles/observability/{{ .Values | get "global.observability" "full" }}/countly.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/countly.yaml - environments/{{ .Environment.Name }}/countly.yaml + - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/countly.yaml - environments/{{ .Environment.Name }}/credentials-countly.yaml needs: - mongodb/countly-mongodb @@ -92,10 +95,10 @@ releases: namespace: observability values: - environments/{{ .Environment.Name }}/global.yaml - - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/observability.yaml - profiles/observability/{{ .Values | get "global.observability" "full" }}/observability.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/observability.yaml - environments/{{ .Environment.Name }}/observability.yaml + - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/observability.yaml - environments/{{ .Environment.Name }}/credentials-observability.yaml needs: - countly/countly diff --git a/scripts/new-argocd-customer.sh b/scripts/new-argocd-customer.sh index a9ea587..bf1c4ee 100755 --- a/scripts/new-argocd-customer.sh +++ b/scripts/new-argocd-customer.sh @@ -25,6 +25,7 @@ Defaults: tls letsencrypt observability full kafkaConnect balanced + kafkaConnectSizing auto migration disabled gcpSA set after scaffold for External Secrets Workload Identity EOF @@ -116,6 +117,7 @@ global: sizing: production observability: full kafkaConnect: balanced + kafkaConnectSizing: auto tls: letsencrypt security: open @@ -347,6 +349,7 @@ security: open tls: letsencrypt observability: full kafkaConnect: balanced +kafkaConnectSizing: auto migration: disabled nginxIngress: service: From e4811706061775181af0bb4c94fb6ef92874db02 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 22:41:35 +0530 Subject: [PATCH 12/15] Fix Kafka ApplicationSet YAML templating --- argocd/applicationsets/02-kafka.yaml | 8 +------- argocd/countly-hosted/applicationsets/02-kafka.yaml | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index fd05f9d..90b693a 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -32,13 +32,7 @@ spec: - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "../../environments/{{ .environment }}/kafka.yaml" - {{- $kcSizing := .kafkaConnectSizing | default "auto" }} - {{- if eq $kcSizing "auto" }} - {{- $kcSizing = .sizing }} - {{- end }} - {{- if $kcSizing }} - - "../../profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml" - {{- end }} + - "../../profiles/kafka-connect-sizing/{{ if or (eq (.kafkaConnectSizing | default \"auto\") \"\") (eq (.kafkaConnectSizing | default \"auto\") \"auto\") }}{{ .sizing }}{{ else }}{{ .kafkaConnectSizing }}{{ end }}/kafka.yaml" - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../environments/{{ .environment }}/credentials-kafka.yaml" parameters: diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 574ad09..950fdb6 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -32,13 +32,7 @@ spec: - "../../profiles/security/{{ .security }}/kafka.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/kafka.yaml" - {{- $kcSizing := .kafkaConnectSizing | default "auto" }} - {{- if eq $kcSizing "auto" }} - {{- $kcSizing = .sizing }} - {{- end }} - {{- if $kcSizing }} - - "../../profiles/kafka-connect-sizing/{{ $kcSizing }}/kafka.yaml" - {{- end }} + - "../../profiles/kafka-connect-sizing/{{ if or (eq (.kafkaConnectSizing | default \"auto\") \"\") (eq (.kafkaConnectSizing | default \"auto\") \"auto\") }}{{ .sizing }}{{ else }}{{ .kafkaConnectSizing }}{{ end }}/kafka.yaml" - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "$values/environments/{{ .environment }}/credentials-kafka.yaml" parameters: From bc164a451f837521a419c55ffa283e7fd698e9a1 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 22:43:12 +0530 Subject: [PATCH 13/15] Fix Kafka ApplicationSet parameter templating --- argocd/applicationsets/02-kafka.yaml | 4 +--- argocd/countly-hosted/applicationsets/02-kafka.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index 90b693a..2a6885f 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -46,10 +46,8 @@ spec: value: "{{ .observability }}" - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - {{- if .kafkaConnectSizing }} - name: global.kafkaConnectSizing - value: "{{ .kafkaConnectSizing }}" - {{- end }} + value: '{{ .kafkaConnectSizing | default "auto" }}' destination: server: "{{ .server }}" namespace: kafka diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 950fdb6..0b55abf 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -46,10 +46,8 @@ spec: value: "{{ .observability }}" - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - {{- if .kafkaConnectSizing }} - name: global.kafkaConnectSizing - value: "{{ .kafkaConnectSizing }}" - {{- end }} + value: '{{ .kafkaConnectSizing | default "auto" }}' - repoURL: https://github.com/Countly/countly-deployment.git targetRevision: countly-hosted-argo-layout ref: values From 7b4d8a994d78ba95931bc4388e7468338debb928 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Fri, 3 Apr 2026 23:18:38 +0530 Subject: [PATCH 14/15] Handle missing Kafka Connect sizing --- argocd/applicationsets/02-kafka.yaml | 4 ++-- argocd/countly-hosted/applicationsets/02-kafka.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml index 2a6885f..93c6a6f 100644 --- a/argocd/applicationsets/02-kafka.yaml +++ b/argocd/applicationsets/02-kafka.yaml @@ -32,7 +32,7 @@ spec: - "../../profiles/observability/{{ .observability }}/kafka.yaml" - "../../profiles/security/{{ .security }}/kafka.yaml" - "../../environments/{{ .environment }}/kafka.yaml" - - "../../profiles/kafka-connect-sizing/{{ if or (eq (.kafkaConnectSizing | default \"auto\") \"\") (eq (.kafkaConnectSizing | default \"auto\") \"auto\") }}{{ .sizing }}{{ else }}{{ .kafkaConnectSizing }}{{ end }}/kafka.yaml" + - "../../profiles/kafka-connect-sizing/{{ $kcSizing := dig \"kafkaConnectSizing\" \"auto\" . }}{{ if or (eq $kcSizing \"\") (eq $kcSizing \"auto\") }}{{ .sizing }}{{ else }}{{ $kcSizing }}{{ end }}/kafka.yaml" - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "../../environments/{{ .environment }}/credentials-kafka.yaml" parameters: @@ -47,7 +47,7 @@ spec: - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - name: global.kafkaConnectSizing - value: '{{ .kafkaConnectSizing | default "auto" }}' + value: '{{ dig "kafkaConnectSizing" "auto" . }}' destination: server: "{{ .server }}" namespace: kafka diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 0b55abf..6395d3f 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -32,7 +32,7 @@ spec: - "../../profiles/security/{{ .security }}/kafka.yaml" - "$values/environments/{{ .environment }}/global.yaml" - "$values/environments/{{ .environment }}/kafka.yaml" - - "../../profiles/kafka-connect-sizing/{{ if or (eq (.kafkaConnectSizing | default \"auto\") \"\") (eq (.kafkaConnectSizing | default \"auto\") \"auto\") }}{{ .sizing }}{{ else }}{{ .kafkaConnectSizing }}{{ end }}/kafka.yaml" + - "../../profiles/kafka-connect-sizing/{{ $kcSizing := dig \"kafkaConnectSizing\" \"auto\" . }}{{ if or (eq $kcSizing \"\") (eq $kcSizing \"auto\") }}{{ .sizing }}{{ else }}{{ $kcSizing }}{{ end }}/kafka.yaml" - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" - "$values/environments/{{ .environment }}/credentials-kafka.yaml" parameters: @@ -47,7 +47,7 @@ spec: - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - name: global.kafkaConnectSizing - value: '{{ .kafkaConnectSizing | default "auto" }}' + value: '{{ dig "kafkaConnectSizing" "auto" . }}' - repoURL: https://github.com/Countly/countly-deployment.git targetRevision: countly-hosted-argo-layout ref: values From 5f544801d8eeab66a2ef30e8afe32ec0253acef2 Mon Sep 17 00:00:00 2001 From: ihaardik Date: Sat, 4 Apr 2026 00:08:29 +0530 Subject: [PATCH 15/15] Point Argo bootstrap branches to main --- argocd/countly-hosted/applicationsets/00-mongodb.yaml | 6 +++--- argocd/countly-hosted/applicationsets/01-clickhouse.yaml | 6 +++--- argocd/countly-hosted/applicationsets/02-kafka.yaml | 6 +++--- argocd/countly-hosted/applicationsets/03-countly.yaml | 6 +++--- argocd/countly-hosted/applicationsets/04-observability.yaml | 6 +++--- argocd/countly-hosted/applicationsets/05-migration.yaml | 6 +++--- argocd/countly-hosted/operators/00-cert-manager.yaml | 2 +- argocd/countly-hosted/operators/01-mongodb-crds.yaml | 2 +- argocd/countly-hosted/operators/02-mongodb-operator.yaml | 2 +- argocd/countly-hosted/operators/03-clickhouse-operator.yaml | 2 +- argocd/countly-hosted/operators/04-strimzi-operator.yaml | 2 +- argocd/countly-hosted/operators/05-nginx-ingress.yaml | 4 ++-- .../operators/06-letsencrypt-prod-issuer-app.yaml | 4 ++-- .../operators/07-external-secrets-operator.yaml | 2 +- .../countly-hosted/operators/08-cluster-secret-store.yaml | 4 ++-- argocd/countly-hosted/root-application.yaml | 2 +- argocd/root-application.yaml | 2 +- 17 files changed, 32 insertions(+), 32 deletions(-) diff --git a/argocd/countly-hosted/applicationsets/00-mongodb.yaml b/argocd/countly-hosted/applicationsets/00-mongodb.yaml index f706932..74e75b1 100644 --- a/argocd/countly-hosted/applicationsets/00-mongodb.yaml +++ b/argocd/countly-hosted/applicationsets/00-mongodb.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: charts/countly-mongodb helm: releaseName: countly-mongodb @@ -40,7 +40,7 @@ spec: - name: global.security value: "{{ .security }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml index ec567ed..fd00315 100644 --- a/argocd/countly-hosted/applicationsets/01-clickhouse.yaml +++ b/argocd/countly-hosted/applicationsets/01-clickhouse.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: charts/countly-clickhouse helm: releaseName: countly-clickhouse @@ -40,7 +40,7 @@ spec: - name: global.security value: "{{ .security }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/02-kafka.yaml b/argocd/countly-hosted/applicationsets/02-kafka.yaml index 6395d3f..41f1d83 100644 --- a/argocd/countly-hosted/applicationsets/02-kafka.yaml +++ b/argocd/countly-hosted/applicationsets/02-kafka.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: charts/countly-kafka helm: releaseName: countly-kafka @@ -49,7 +49,7 @@ spec: - name: global.kafkaConnectSizing value: '{{ dig "kafkaConnectSizing" "auto" . }}' - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/03-countly.yaml b/argocd/countly-hosted/applicationsets/03-countly.yaml index 0c0def9..720ec65 100644 --- a/argocd/countly-hosted/applicationsets/03-countly.yaml +++ b/argocd/countly-hosted/applicationsets/03-countly.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: charts/countly helm: releaseName: countly @@ -52,7 +52,7 @@ spec: - name: global.kafkaConnect value: "{{ .kafkaConnect }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/04-observability.yaml b/argocd/countly-hosted/applicationsets/04-observability.yaml index 41a0da9..abd39af 100644 --- a/argocd/countly-hosted/applicationsets/04-observability.yaml +++ b/argocd/countly-hosted/applicationsets/04-observability.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: '{{ if eq .observability "disabled" }}charts/noop{{ else }}charts/countly-observability{{ end }}' helm: releaseName: countly-observability @@ -43,7 +43,7 @@ spec: - name: global.observability value: "{{ .observability }}" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/applicationsets/05-migration.yaml b/argocd/countly-hosted/applicationsets/05-migration.yaml index b98f485..de2f277 100644 --- a/argocd/countly-hosted/applicationsets/05-migration.yaml +++ b/argocd/countly-hosted/applicationsets/05-migration.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: "{{ .project }}" sources: - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: '{{ if eq .migration "enabled" }}charts/countly-migration{{ else }}charts/noop{{ end }}' helm: releaseName: countly-migration @@ -34,7 +34,7 @@ spec: - name: argocd.enabled value: "true" - repoURL: https://github.com/Countly/countly-deployment.git - targetRevision: countly-hosted-argo-layout + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/operators/00-cert-manager.yaml b/argocd/countly-hosted/operators/00-cert-manager.yaml index fb23812..1c85565 100644 --- a/argocd/countly-hosted/operators/00-cert-manager.yaml +++ b/argocd/countly-hosted/operators/00-cert-manager.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/01-mongodb-crds.yaml b/argocd/countly-hosted/operators/01-mongodb-crds.yaml index b384810..7a0b6fc 100644 --- a/argocd/countly-hosted/operators/01-mongodb-crds.yaml +++ b/argocd/countly-hosted/operators/01-mongodb-crds.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/02-mongodb-operator.yaml b/argocd/countly-hosted/operators/02-mongodb-operator.yaml index c527a94..db5a16b 100644 --- a/argocd/countly-hosted/operators/02-mongodb-operator.yaml +++ b/argocd/countly-hosted/operators/02-mongodb-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml index 0f7b9e3..e968898 100644 --- a/argocd/countly-hosted/operators/03-clickhouse-operator.yaml +++ b/argocd/countly-hosted/operators/03-clickhouse-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/04-strimzi-operator.yaml b/argocd/countly-hosted/operators/04-strimzi-operator.yaml index be69198..5a0ca16 100644 --- a/argocd/countly-hosted/operators/04-strimzi-operator.yaml +++ b/argocd/countly-hosted/operators/04-strimzi-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/05-nginx-ingress.yaml b/argocd/countly-hosted/operators/05-nginx-ingress.yaml index a513d55..4979b66 100644 --- a/argocd/countly-hosted/operators/05-nginx-ingress.yaml +++ b/argocd/countly-hosted/operators/05-nginx-ingress.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -36,7 +36,7 @@ spec: loadBalancerIP: {{ $lbIP | quote }} {{- end }} - repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main ref: values destination: server: "{{ .server }}" diff --git a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml index 74533f0..23190c9 100644 --- a/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml +++ b/argocd/countly-hosted/operators/06-letsencrypt-prod-issuer-app.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: '{{ if eq .tls "letsencrypt" }}argocd/operator-manifests/letsencrypt-prod-issuer{{ else }}charts/noop{{ end }}' directory: recurse: true diff --git a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml index 5cfb75c..b8c323c 100644 --- a/argocd/countly-hosted/operators/07-external-secrets-operator.yaml +++ b/argocd/countly-hosted/operators/07-external-secrets-operator.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: diff --git a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml index 0aa9a1c..520d5bc 100644 --- a/argocd/countly-hosted/operators/08-cluster-secret-store.yaml +++ b/argocd/countly-hosted/operators/08-cluster-secret-store.yaml @@ -10,7 +10,7 @@ spec: generators: - git: repoURL: https://github.com/Countly/countly-deployment.git - revision: countly-hosted-argo-layout + revision: main files: - path: customers/*.yaml template: @@ -22,7 +22,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: charts/countly-cluster-secret-store helm: releaseName: countly-cluster-secret-store diff --git a/argocd/countly-hosted/root-application.yaml b/argocd/countly-hosted/root-application.yaml index e5fd85a..ffce1cc 100644 --- a/argocd/countly-hosted/root-application.yaml +++ b/argocd/countly-hosted/root-application.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: argocd/countly-hosted directory: recurse: true diff --git a/argocd/root-application.yaml b/argocd/root-application.yaml index 25db1a5..8382d9d 100644 --- a/argocd/root-application.yaml +++ b/argocd/root-application.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/Countly/helm.git - targetRevision: mutli-source-values-split + targetRevision: main path: argocd directory: recurse: true