diff --git a/.badges/operations.svg b/.badges/operations.svg index f29f7c0811..b0f0452890 100644 --- a/.badges/operations.svg +++ b/.badges/operations.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ PARITY entries PARITY entries - 9318 - 9318 + 9494 + 9494 diff --git a/.badges/parity.svg b/.badges/parity.svg index ebf24c67a5..0d5002ae3c 100644 --- a/.badges/parity.svg +++ b/.badges/parity.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ parity parity - 169 A · 87 B · 3 C - 169 A · 87 B · 3 C + 169 A · 91 B · 3 C + 169 A · 91 B · 3 C diff --git a/.badges/services.svg b/.badges/services.svg index 69b929ea37..ebdfa36acc 100644 --- a/.badges/services.svg +++ b/.badges/services.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ AWS services AWS services - 261 - 261 + 265 + 265 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a24239cd7..fbaaf6664a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -862,6 +862,34 @@ jobs: GOPHERSTACK_TEST_IMAGE: gopherstack:tc-local run: go test -count=1 -timeout 10m ./... + helm-tests: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ env.GH_CI_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + check-latest: true + cache: true + + - name: Download binary + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: gopherstack-binary + path: bin/ + + - name: Lint, render and install the Helm chart on k3s + env: + GOPHERSTACK_TEST_DOCKER: "1" + run: | + chmod +x bin/gopherstack-linux + go test -count=1 -timeout 15m -v ./test/helm/ + integration: runs-on: ubuntu-latest needs: [integration-tests] diff --git a/Dockerfile b/Dockerfile index 05ebb2a088..dd996a1267 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,13 +32,22 @@ RUN CGO_ENABLED=0 go build \ -ldflags="-w -s -extldflags '-static -fno-PIC'" \ -o gopherstack -# Final stage -FROM scratch +# Writable state dir for the non-root runtime user (65532:65532) +FROM --platform=$BUILDPLATFORM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data -WORKDIR /root/ +# Final stage: distroless static (nonroot) has a world-searchable / (containerd would unpack a bare +# scratch / as 0700 root), plus ca-certificates, tzdata and a 1777 /tmp. +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 + +COPY --from=state --chown=65532:65532 /data /data + +WORKDIR /usr/local/bin + +COPY --from=builder --chown=0:0 --chmod=755 /app/gopherstack . + +USER 65532:65532 -# Copy the Pre-built binary from the previous stage -COPY --from=builder /app/gopherstack . # Expose port 8000 to the outside world EXPOSE 8000 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index e9789d37eb..6f28a4b006 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,12 +1,21 @@ # Dockerfile used by GoReleaser for multi-arch builds -FROM scratch +FROM --platform=$BUILDPLATFORM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data + +# distroless static (nonroot): world-searchable /, ca-certificates, tzdata, 1777 /tmp, UID 65532. +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 ARG TARGETPLATFORM -WORKDIR /root/ +COPY --from=state --chown=65532:65532 /data /data + +WORKDIR /usr/local/bin # Copy the binary from GoReleaser context -COPY $TARGETPLATFORM/gopherstack . +COPY --chown=0:0 --chmod=755 $TARGETPLATFORM/gopherstack . + +USER 65532:65532 + # Expose port 8000 EXPOSE 8000 diff --git a/Dockerfile.test b/Dockerfile.test index 2e28f8490a..e1a041e359 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,8 +1,14 @@ -# Lightweight test image — expects a pre-built static binary at bin/gopherstack-linux -# (from `task build-linux`; this is a separate path from `task build`'s bin/gopherstack -# so the two targets can't clobber each other — see gopherstack-gooq). -FROM scratch -COPY bin/gopherstack-linux /root/gopherstack +# Lightweight test image: expects a pre-built static binary at bin/gopherstack-linux +# (from `task build-linux`; separate from `task build`'s bin/gopherstack — see gopherstack-gooq). +# Same non-root layout as the release image. Must build with the classic (non-BuildKit) +# builder testcontainers uses: no $BUILDPLATFORM, no COPY --chmod (the binary is already 0755). +FROM alpine:3.22@sha256:2e0b81d60597f449baa56dc25d355367869d2dca4cd57e866f1e2b52af3eb55f AS state +RUN mkdir /data + +FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329 +COPY --from=state --chown=65532:65532 /data /data +COPY --chown=0:0 bin/gopherstack-linux /usr/local/bin/gopherstack +USER 65532:65532 EXPOSE 8000 EXPOSE 1883 -CMD ["/root/gopherstack"] +CMD ["/usr/local/bin/gopherstack"] diff --git a/README.md b/README.md index 0714e3ef8d..9a3afee6a8 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,14 @@ docker compose up -d For Lambda support the container needs access to a container runtime — see [docs/docker.md](docs/docker.md). +### Kubernetes (Helm) + +```bash +helm install gopherstack ./charts/gopherstack +``` + +See [docs/kubernetes.md](docs/kubernetes.md). + ### Testcontainers (Go) Gopherstack ships a reusable [Testcontainers for Go](https://golang.testcontainers.org/) @@ -339,6 +347,7 @@ A single switch picks the default for every container-backed engine (RDS/Aurora, | `--transparent-ca-dir` | `TRANSPARENT_CA_DIR` | ephemeral | Directory to persist the CA so it is trusted once. | | `--transparent-container-ca-path` | `TRANSPARENT_CONTAINER_CA_PATH` | temp file (skipped when containerised) | CA PEM path visible to the Docker daemon; required to inject the CA when gopherstack itself runs in Docker. | | `--rds-engine` | `RDS_ENGINE` | `stub` | RDS engine mode: `stub` (metadata only) or `docker` (real `postgres`/`mysql`/`mariadb` container per DB instance, one per Aurora cluster; Data API stays on SQLite; `RDS_DB_HOST` overrides the advertised host, default `127.0.0.1`; ports bind loopback only unless it is non-loopback, then all interfaces; Postgres runs `postgis/postgis` + pgvector with RDS-style `aws_s3`/`aws_lambda` extensions unless `RDS_POSTGRES_EXTENSIONS=false`; `RDS_SQLSERVER_ENGINE=docker` opts `sqlserver-*` into a real SQL Server container (amd64 only; accepts the Microsoft EULA); `RDS_CALLOUT_HOST`/`RDS_CALLOUT_BIND` set the address containers dial / the callout listener binds (default: container bridge gateway, never a wildcard)). | +| `--dsql-engine` | `DSQL_ENGINE` | `stub` | Aurora DSQL engine mode: `stub` (control plane only) or `docker` (one `postgres` container per cluster behind a TLS proxy that accepts `GenerateDbConnectAuthToken` / `GenerateDBConnectAdminAuthToken` tokens as the password for user `admin`; `CREATE INDEX ASYNC`, `sys.jobs`, `sys.wait_for_job`; connect address at `/_gopherstack/dsql/clusters/{id}`, CA at `/_gopherstack/dsql/ca.pem`; `DSQL_ENGINE_IMAGE` picks the image, default `postgres:17`). | | `--docdb-engine` | `DOCDB_ENGINE` | `stub` | DocumentDB engine mode: `stub` (metadata only) or `docker` (one plaintext `mongo` container per cluster, `mongo:4.4` for 3.6/4.0 and `mongo:5.0` for 5.0; `DOCDB_HOST` overrides the advertised host, default `127.0.0.1`; ports bind loopback only unless it is non-loopback, then all interfaces). | | `--redshift-engine` | `REDSHIFT_ENGINE` | `stub` | Redshift engine mode: `stub` (metadata only) or `docker` (one `postgres:17` container per provisioned cluster with MasterUsername/MasterUserPassword/DBName; Endpoint reports the published `127.0.0.1` port and the Redshift Data API runs SQL against it; `REDSHIFT_HOST` overrides the advertised host; Serverless workgroups stay metadata-only). | | `--neptune-engine` | `NEPTUNE_ENGINE` | `stub` | Neptune engine mode: `stub` (metadata only), `embedded` (in-process Gremlin, openCypher and Bolt over one persisted property graph; no container runtime) or `docker` (the same in-process graph by default; `NEPTUNE_GREMLIN=container` runs one `tinkerpop/gremlin-server` container per cluster instead, TinkerPop 3.4.13 for engine 1.0/1.1, 3.5.7 for 1.2, 3.6.8 for 1.3, 3.7.3 later). Endpoint, ReaderEndpoint, Port and member instances report the published `127.0.0.1` port, Gremlin over WebSocket/HTTP at `/gremlin`; `NEPTUNE_HOST` overrides the advertised host; no IAM auth. | @@ -890,7 +899,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [SageMaker](services/sagemaker/README.md) | A | 70 | 1 gap; 19 structural gaps | | [SageMaker Runtime](services/sagemakerruntime/README.md) | A | 3 | 1 gap; 1 structural gap | | [Textract](services/textract/README.md) | A | 25 | 2 structural gaps; 1 deferred | -| [Transcribe](services/transcribe/README.md) | A | 44 | 2 structural gaps | +| [Transcribe](services/transcribe/README.md) | A | 44 | 6 gaps; 2 structural gaps | | [Translate](services/translate/README.md) | A | 19 | 1 gap; 2 structural gaps | ### Media @@ -969,7 +978,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Google Cloud KMS](services/gcpkms/README.md) | B | 35 | 4 gaps | | [Google Cloud Pub/Sub](services/gcppubsub/README.md) | B | 38 | 6 gaps | | [Google Cloud Resource Manager and Service Usage](services/gcpresourcemanager/README.md) | B | 39 | 9 gaps | -| [Google Cloud Storage](services/gcpstorage/README.md) | B | 28 | 8 gaps | +| [Google Cloud Storage](services/gcpstorage/README.md) | B | 37 | 9 gaps | | [Google Secret Manager](services/gcpsecretmanager/README.md) | B | 18 | 5 gaps | | [Identity Platform](services/gcpidentitytoolkit/README.md) | B | 1 | 6 gaps | | [Kubernetes Engine](services/gcpcontainer/README.md) | B | — | 5 families; 5 gaps | @@ -982,10 +991,13 @@ Every service links to its own page with a coverage breakdown — audited operat |---|---|---|---| | [OCI API Gateway](services/ociapigateway/README.md) | B | 22 | 10 gaps | | [OCI Artifacts and Container Registry](services/ociartifacts/README.md) | B | 37 | 7 gaps | +| [OCI Autonomous Database](services/ocidatabase/README.md) | B | 21 | 7 gaps | | [OCI Container Engine for Kubernetes](services/ocicontainerengine/README.md) | B | 20 | 9 gaps | | [OCI Container Instances](services/ocicontainerinstances/README.md) | B | 18 | 7 gaps | | [OCI Core Services](services/ocicore/README.md) | B | 135 | 11 gaps | +| [OCI DNS](services/ocidns/README.md) | B | 52 | 7 gaps | | [OCI Events](services/ocievents/README.md) | B | 6 | 6 gaps | +| [OCI File Storage](services/ocifilestorage/README.md) | B | 74 | 7 gaps | | [OCI Functions](services/ocifunctions/README.md) | B | 16 | 9 gaps | | [OCI Identity](services/ociidentity/README.md) | B | 70 | 9 gaps | | [OCI Load Balancer](services/ociloadbalancer/README.md) | B | 61 | 9 gaps | @@ -1006,7 +1018,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Amp](services/amp/README.md) | B | 44 | 9 gaps; 2 structural gaps | | [AppStream 2.0](services/appstream/README.md) | A | 44 | 5 structural gaps | | [Appintegrations](services/appintegrations/README.md) | B | 23 | 3 gaps; 1 structural gap | -| [Azurearm](services/azurearm/README.md) | B | 20 | 69 gaps; 3 deferred | +| [Azurearm](services/azurearm/README.md) | B | 20 | 73 gaps; 3 deferred | | [Azurestoragevhost](services/azurestoragevhost/README.md) | B | 2 | 2 gaps; 1 deferred | | [Bcmdataexports](services/bcmdataexports/README.md) | B | 12 | 6 gaps | | [Bedrockagentcore](services/bedrockagentcore/README.md) | B | 30 | 11 gaps; 2 structural gaps | @@ -1018,13 +1030,14 @@ Every service links to its own page with a coverage breakdown — audited operat | [Connect](services/connect/README.md) | B | 312 | 21 gaps; 2 structural gaps | | [Cur](services/cur/README.md) | B | 7 | 6 gaps | | [Directconnect](services/directconnect/README.md) | A | 73 | 2 gaps; 8 structural gaps; 1 deferred | -| [Dsql](services/dsql/README.md) | A | 16 | 3 structural gaps | +| [Dsql](services/dsql/README.md) | A | 16 | 8 gaps; 2 structural gaps | | [Ecrpublic](services/ecrpublic/README.md) | A | 23 | 3 structural gaps | | [Globalaccelerator](services/globalaccelerator/README.md) | B | 56 | 4 gaps; 2 structural gaps | | [Grafana](services/grafana/README.md) | A | 25 | 1 structural gap | | [HealthOmics](services/omics/README.md) | A | — | 25 families; 5 structural gaps; 1 deferred | | [Kafkaconnect](services/kafkaconnect/README.md) | A | 19 | 3 gaps; 3 structural gaps | | [Kinesisvideo](services/kinesisvideo/README.md) | A | 32 | 3 structural gaps | +| [Lambdaweb](services/lambdaweb/README.md) | B | 20 | 11 gaps; 1 structural gap | | [Lightsail](services/lightsail/README.md) | A | — | 29 families; 1 gap; 6 structural gaps; 2 deferred | | [Managed Blockchain](services/managedblockchain/README.md) | A | 27 | 3 structural gaps | | [Marketplaceentitlementservice](services/marketplaceentitlementservice/README.md) | B | 1 | 3 gaps; 1 structural gap | @@ -1148,6 +1161,7 @@ cdk deploy - [Quickstart](docs/quickstart.md) - [Running with Docker](docs/docker.md) +- [Running on Kubernetes (Helm)](docs/kubernetes.md) - [Security group enforcement](docs/security-groups.md) - [Migrating from LocalStack](docs/migration.md) - [Architecture](docs/architecture/README.md) diff --git a/Taskfile.yml b/Taskfile.yml index eecd1b307a..1f7b6441b0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -165,6 +165,21 @@ tasks: -ldflags "-w -s -X {{.VERSION_PKG}}.Build={{.BUILD_VERSION}}" \ -o bin/{{.BINARY_NAME}}-linux . + helm-test: + desc: Lint and golden-render charts/gopherstack with the pinned helm image (needs docker) + env: + GOPHERSTACK_TEST_DOCKER: "1" + cmds: + - go test -count=1 -run 'TestChart_(Lint|GoldenTemplate|SchemaRejects)' ./test/helm/ + + helm-install-test: + desc: Install charts/gopherstack into a k3s cluster and call it (needs docker) + deps: [build-linux] + env: + GOPHERSTACK_TEST_DOCKER: "1" + cmds: + - go test -count=1 -timeout 15m -run TestChart_InstallK3s -v ./test/helm/ + install-deps: desc: Install pinned golangci-lint, fieldalignment and dev MCP CLIs run: once diff --git a/charts/gopherstack/.helmignore b/charts/gopherstack/.helmignore new file mode 100644 index 0000000000..483618d6ec --- /dev/null +++ b/charts/gopherstack/.helmignore @@ -0,0 +1,4 @@ +.git +*.swp +*.bak +*.tmp diff --git a/charts/gopherstack/Chart.yaml b/charts/gopherstack/Chart.yaml new file mode 100644 index 0000000000..805c6466d9 --- /dev/null +++ b/charts/gopherstack/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: gopherstack +description: Gopherstack, a local multi-cloud (AWS, Azure, GCP, OCI) service emulator +type: application +version: 0.1.0 +appVersion: "1.9.1" +kubeVersion: ">=1.25.0-0" +home: https://github.com/blackbirdworks/gopherstack +sources: + - https://github.com/blackbirdworks/gopherstack +keywords: + - aws + - emulator + - localstack + - testing diff --git a/charts/gopherstack/templates/NOTES.txt b/charts/gopherstack/templates/NOTES.txt new file mode 100644 index 0000000000..1b3b2829d7 --- /dev/null +++ b/charts/gopherstack/templates/NOTES.txt @@ -0,0 +1,29 @@ +Gopherstack {{ .Chart.AppVersion }} is deploying as {{ include "gopherstack.fullname" . }}. + +Point AWS SDKs and the CLI at the service: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + export AWS_ENDPOINT_URL=http{{ if $.Values.ingress.tls }}s{{ end }}://{{ .host }} +{{- end }} +{{- else if eq .Values.service.type "NodePort" }} + export NODE_PORT=$(kubectl get svc {{ include "gopherstack.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}') + export AWS_ENDPOINT_URL=http://:$NODE_PORT +{{- else if eq .Values.service.type "LoadBalancer" }} + export AWS_ENDPOINT_URL=http://$(kubectl get svc {{ include "gopherstack.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):{{ .Values.service.port }} +{{- else }} + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "gopherstack.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.port }} + export AWS_ENDPOINT_URL=http://localhost:{{ .Values.service.port }} +{{- end }} + export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION={{ .Values.config.region }} + +In-cluster endpoint: http://{{ include "gopherstack.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }} +Health: {{ .Values.probes.path }} +{{- if not .Values.persistence.enabled }} + +State is in memory and is lost when the pod restarts. Set persistence.enabled=true to keep it. +{{- end }} +{{- if not .Values.dind.enabled }} + +Docker-backed engines (RDS, EKS, Lambda containers, ...) need a container runtime and run +as stubs in this pod. Set dind.enabled=true and config.engines=docker to use a DinD sidecar. +{{- end }} diff --git a/charts/gopherstack/templates/_helpers.tpl b/charts/gopherstack/templates/_helpers.tpl new file mode 100644 index 0000000000..3d25b70bf8 --- /dev/null +++ b/charts/gopherstack/templates/_helpers.tpl @@ -0,0 +1,48 @@ +{{- define "gopherstack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "gopherstack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "gopherstack.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "gopherstack.labels" -}} +helm.sh/chart: {{ include "gopherstack.chart" . }} +{{ include "gopherstack.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "gopherstack.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gopherstack.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "gopherstack.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "gopherstack.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{- define "gopherstack.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) -}} +{{- end -}} + +{{- define "gopherstack.pvcName" -}} +{{- default (include "gopherstack.fullname" .) .Values.persistence.existingClaim -}} +{{- end -}} diff --git a/charts/gopherstack/templates/configmap.yaml b/charts/gopherstack/templates/configmap.yaml new file mode 100644 index 0000000000..79fc7aa638 --- /dev/null +++ b/charts/gopherstack/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +data: + PORT: {{ .Values.service.port | quote }} + REGION: {{ .Values.config.region | quote }} + ACCOUNT_ID: {{ .Values.config.accountId | quote }} + LOG_LEVEL: {{ .Values.config.logLevel | quote }} + GOPHERSTACK_ENGINES: {{ .Values.config.engines | quote }} + GOPHERSTACK_ENFORCE_IAM: {{ .Values.config.enforceIam | quote }} + GOPHERSTACK_LOCALSTACK_COMPAT: {{ .Values.config.localstackCompat | quote }} + {{- with .Values.config.services }} + GOPHERSTACK_SERVICES: {{ join "," . | quote }} + {{- end }} + {{- with .Values.config.disableServices }} + GOPHERSTACK_DISABLE_SERVICES: {{ join "," . | quote }} + {{- end }} + {{- if .Values.persistence.enabled }} + PERSIST: "true" + GOPHERSTACK_DATA_DIR: {{ .Values.persistence.mountPath | quote }} + {{- end }} + {{- if .Values.dind.enabled }} + DOCKER_HOST: "tcp://127.0.0.1:2375" + {{- end }} + {{- range $k, $v := .Values.config.extra }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/charts/gopherstack/templates/deployment.yaml b/charts/gopherstack/templates/deployment.yaml new file mode 100644 index 0000000000..7a4b9d5b0f --- /dev/null +++ b/charts/gopherstack/templates/deployment.yaml @@ -0,0 +1,125 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "gopherstack.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gopherstack.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gopherstack.serviceAccountName" . }} + automountServiceAccountToken: false + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: gopherstack + image: {{ include "gopherstack.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + envFrom: + - configMapRef: + name: {{ include "gopherstack.fullname" . }} + {{- with .Values.envFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + {{- range .Values.service.extraPorts }} + - name: {{ .name }} + containerPort: {{ .port }} + protocol: {{ default "TCP" .protocol }} + {{- end }} + livenessProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.liveness | nindent 12 }} + readinessProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.readiness | nindent 12 }} + startupProbe: + httpGet: + path: {{ .Values.probes.path }} + port: http + {{- toYaml .Values.probes.startup | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- if .Values.dind.enabled }} + - name: dind + image: {{ printf "%s:%s" .Values.dind.image.repository .Values.dind.image.tag | quote }} + imagePullPolicy: {{ .Values.dind.image.pullPolicy }} + args: ["--host=tcp://127.0.0.1:2375"] + env: + - name: DOCKER_TLS_CERTDIR + value: "" + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + resources: + {{- toYaml .Values.dind.resources | nindent 12 }} + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "gopherstack.pvcName" . }} + {{- end }} + {{- if .Values.dind.enabled }} + - name: dind-storage + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/gopherstack/templates/ingress.yaml b/charts/gopherstack/templates/ingress.yaml new file mode 100644 index 0000000000..85c3425799 --- /dev/null +++ b/charts/gopherstack/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ default "Prefix" .pathType }} + backend: + service: + name: {{ include "gopherstack.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/gopherstack/templates/pvc.yaml b/charts/gopherstack/templates/pvc.yaml new file mode 100644 index 0000000000..b03f5251af --- /dev/null +++ b/charts/gopherstack/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- with .Values.persistence.storageClass }} + storageClassName: {{ if eq . "-" }}""{{ else }}{{ . | quote }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- end }} diff --git a/charts/gopherstack/templates/service.yaml b/charts/gopherstack/templates/service.yaml new file mode 100644 index 0000000000..fc6b33c6d0 --- /dev/null +++ b/charts/gopherstack/templates/service.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gopherstack.fullname" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- range .Values.service.extraPorts }} + - name: {{ .name }} + port: {{ .port }} + targetPort: {{ .name }} + protocol: {{ default "TCP" .protocol }} + {{- if and (eq $.Values.service.type "NodePort") .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + {{- end }} + selector: + {{- include "gopherstack.selectorLabels" . | nindent 4 }} diff --git a/charts/gopherstack/templates/serviceaccount.yaml b/charts/gopherstack/templates/serviceaccount.yaml new file mode 100644 index 0000000000..c07a58a529 --- /dev/null +++ b/charts/gopherstack/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gopherstack.serviceAccountName" . }} + labels: + {{- include "gopherstack.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/charts/gopherstack/values.schema.json b/charts/gopherstack/values.schema.json new file mode 100644 index 0000000000..f5e9d16885 --- /dev/null +++ b/charts/gopherstack/values.schema.json @@ -0,0 +1,138 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "image": { + "type": "object", + "required": ["repository"], + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "imagePullSecrets": {"type": "array"}, + "nameOverride": {"type": "string"}, + "fullnameOverride": {"type": "string"}, + "config": { + "type": "object", + "properties": { + "region": {"type": "string", "minLength": 1}, + "accountId": {"type": "string", "pattern": "^[0-9]{12}$"}, + "logLevel": {"enum": ["debug", "info", "warn", "error"]}, + "engines": {"enum": ["stub", "auto", "docker"]}, + "enforceIam": {"type": "boolean"}, + "localstackCompat": {"type": "boolean"}, + "services": {"type": "array", "items": {"type": "string"}}, + "disableServices": {"type": "array", "items": {"type": "string"}}, + "extra": {"type": "object", "additionalProperties": {"type": ["string", "number", "boolean"]}} + } + }, + "env": {"type": "array"}, + "envFrom": {"type": "array"}, + "service": { + "type": "object", + "properties": { + "type": {"enum": ["ClusterIP", "NodePort", "LoadBalancer"]}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "nodePort": {"type": ["integer", "null"], "minimum": 30000, "maximum": 32767}, + "annotations": {"type": "object"}, + "extraPorts": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "port"], + "properties": { + "name": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "maxLength": 15}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, + "protocol": {"enum": ["TCP", "UDP"]}, + "nodePort": {"type": "integer", "minimum": 30000, "maximum": 32767} + } + } + } + } + }, + "ingress": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "className": {"type": "string"}, + "annotations": {"type": "object"}, + "hosts": { + "type": "array", + "items": { + "type": "object", + "required": ["host", "paths"], + "properties": { + "host": {"type": "string"}, + "paths": { + "type": "array", + "items": { + "type": "object", + "required": ["path"], + "properties": { + "path": {"type": "string"}, + "pathType": {"enum": ["Prefix", "Exact", "ImplementationSpecific"]} + } + } + } + } + } + }, + "tls": {"type": "array"} + } + }, + "persistence": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "size": {"type": "string"}, + "storageClass": {"type": "string"}, + "accessModes": {"type": "array", "items": {"type": "string"}}, + "existingClaim": {"type": "string"}, + "mountPath": {"type": "string", "pattern": "^/"}, + "annotations": {"type": "object"} + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": {"type": "boolean"}, + "name": {"type": "string"}, + "annotations": {"type": "object"} + } + }, + "dind": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "image": { + "type": "object", + "properties": { + "repository": {"type": "string"}, + "tag": {"type": "string"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "resources": {"type": "object"} + } + }, + "probes": { + "type": "object", + "properties": { + "path": {"type": "string", "pattern": "^/"}, + "liveness": {"type": "object"}, + "readiness": {"type": "object"}, + "startup": {"type": "object"} + } + }, + "podAnnotations": {"type": "object"}, + "podLabels": {"type": "object"}, + "podSecurityContext": {"type": "object"}, + "securityContext": {"type": "object"}, + "resources": {"type": "object"}, + "nodeSelector": {"type": "object"}, + "tolerations": {"type": "array"}, + "affinity": {"type": "object"} + } +} diff --git a/charts/gopherstack/values.yaml b/charts/gopherstack/values.yaml new file mode 100644 index 0000000000..da29aff8a8 --- /dev/null +++ b/charts/gopherstack/values.yaml @@ -0,0 +1,112 @@ +image: + repository: ghcr.io/blackbirdworks/gopherstack + # Defaults to the chart appVersion. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +# Gopherstack keeps state in one process; the Deployment always runs one replica. +config: + region: us-east-1 + accountId: "000000000000" + logLevel: info + # Engine default for container-backed services: stub | auto | docker. + engines: stub + enforceIam: false + localstackCompat: false + # Only enable / disable these services (empty services = all). + services: [] + disableServices: [] + # Any other environment variable, as KEY: value. + extra: {} + +env: [] +envFrom: [] + +service: + type: ClusterIP + port: 8000 + nodePort: null + annotations: {} + # Dedicated per-cloud listeners, e.g. Azure Blob 10000, Service Bus 10003, Cosmos 8081, + # GCP Pub/Sub 8085, Storage 4443. Each entry needs the matching *_PORT env var if it is not the default. + extraPorts: [] + # - name: azure-blob + # port: 10000 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: gopherstack.local + paths: + - path: / + pathType: Prefix + tls: [] + +persistence: + enabled: false + size: 1Gi + storageClass: "" + accessModes: + - ReadWriteOnce + existingClaim: "" + mountPath: /data + annotations: {} + +serviceAccount: + create: true + name: "" + annotations: {} + +# Docker-in-Docker sidecar so docker-backed engines work in a cluster. Runs privileged. +dind: + enabled: false + image: + repository: docker + tag: 27-dind + pullPolicy: IfNotPresent + resources: {} + +probes: + path: /_gopherstack/health + liveness: + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + readiness: + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + startup: + periodSeconds: 2 + timeoutSeconds: 3 + failureThreshold: 60 + +podAnnotations: {} +podLabels: {} + +# The release image runs as UID/GID 65532 and owns /data; fsGroup makes a PVC writable by that user. +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + +resources: {} + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/cmd/gendocs/categories.go b/cmd/gendocs/categories.go index 873a2f8d1b..6a2156d4bd 100644 --- a/cmd/gendocs/categories.go +++ b/cmd/gendocs/categories.go @@ -53,6 +53,9 @@ const ( slugOCILoadBalancer = "ociloadbalancer" slugOCIArtifacts = "ociartifacts" slugOCIInstances = "ocicontainerinstances" + slugOCIDNS = "ocidns" + slugOCIDatabase = "ocidatabase" + slugOCIFileStorage = "ocifilestorage" slugGCPPubSub = "gcppubsub" slugGCPSecretMgr = "gcpsecretmanager" slugGCPKMS = "gcpkms" @@ -96,7 +99,7 @@ func ociSlugs() []string { slugOCIObjectStore, slugOCIVault, slugOCIStreaming, slugOCIIdentity, slugOCIFunctions, slugOCICore, slugOCIQueue, slugOCINotify, slugOCIContainerEng, slugOCIMySQL, slugOCILogging, slugOCIMonitoring, slugOCINoSQL, slugOCIAPIGateway, slugOCIEvents, slugOCILoadBalancer, slugOCIArtifacts, - slugOCIInstances, + slugOCIInstances, slugOCIDNS, slugOCIDatabase, slugOCIFileStorage, } } @@ -326,6 +329,9 @@ func displayNamesB() map[string]string { slugOCILoadBalancer: "OCI Load Balancer", slugOCIArtifacts: "OCI Artifacts and Container Registry", slugOCIInstances: "OCI Container Instances", + slugOCIDNS: "OCI DNS", + slugOCIDatabase: "OCI Autonomous Database", + slugOCIFileStorage: "OCI File Storage", slugGCPPubSub: "Google Cloud Pub/Sub", slugGCPSecretMgr: "Google Secret Manager", slugGCPKMS: "Google Cloud KMS", diff --git a/dashboard/azurearm_inproc_proxy.go b/dashboard/azurearm_inproc_proxy.go index 6b036ebd61..c746846fb8 100644 --- a/dashboard/azurearm_inproc_proxy.go +++ b/dashboard/azurearm_inproc_proxy.go @@ -41,8 +41,10 @@ type armProxy struct { view func(doc any) name string prefix string + version string queryKeys []string methods []string + relayOps bool } func (h *DashboardHandler) registerARMProxy(px armProxy) { @@ -73,6 +75,10 @@ func (h *DashboardHandler) registerARMProxy(px armProxy) { } } + if px.version != "" { + q.Set(armQueryVersion, px.version) + } + fwd := r.Clone(r.Context()) fwd.URL.Path, fwd.URL.RawPath = unescapedPath(armPath), armPath fwd.URL.RawQuery = q.Encode() @@ -91,6 +97,10 @@ func (h *DashboardHandler) registerARMProxy(px armProxy) { rec := httptest.NewRecorder() inner.ServeHTTP(rec, fwd) + if px.relayOps { + netRelayOps(c, rec.Header(), px.prefix) + } + return armProxyRelay(c, rec, px.view) } diff --git a/dashboard/azureauthz_proxy.go b/dashboard/azureauthz_proxy.go new file mode 100644 index 0000000000..66b5027d18 --- /dev/null +++ b/dashboard/azureauthz_proxy.go @@ -0,0 +1,165 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "regexp" + "slices" + "strings" +) + +const ( + azureAuthzPrefix = "/dashboard/api/v1/azure/authz" + authzPxSubScope = 2 + authzPxRGScope = 4 + authzPxNS = "microsoft.authorization" + authzPxRoleDefs = "roledefinitions" + authzPxRoleAssign = "roleassignments" + authzPxLocks = "locks" + authzPxMaxNotes = 512 +) + +//nolint:gochecknoglobals // immutable allow-lists and validators +var ( + authzPxReadOnly = []string{"policydefinitions", "policysetdefinitions", "policyassignments", authzPxRoleDefs} + authzPxLevels = []string{"CanNotDelete", "ReadOnly"} + authzPxTypes = []string{"User", "Group", "ServicePrincipal"} + authzPxLockName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,89}$`) + authzPxRoleDef = regexp.MustCompile( + `^/subscriptions/[^/]+/providers/microsoft\.authorization/roledefinitions/[0-9a-f-]{36}$`) +) + +// registerAzureAuthzRoutes serves the access half of the Entra page: role definitions and assignments, locks and +// policy at subscription or resource-group scope, by dispatching in-process to the ARM handler. Role assignment and +// lock writes are rebuilt from fixed fields; policy and role definitions are read-only. +func (h *DashboardHandler) registerAzureAuthzRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure Authorization", prefix: azureAuthzPrefix, + queryKeys: []string{armQueryVersion, "$filter"}, + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + allowed: authzProxyAllowed, + body: authzProxyBody, + }) +} + +// authzSplit splits a lower-cased path ({scope}/providers/microsoft.authorization/{type}[/{name}]) where the scope +// is a subscription or a resource group. +func authzSplit(p []string) (string, string, bool) { + scope := authzPxSubScope + if len(p) > authzPxSubScope && p[0] == armPxSubsSeg && p[2] == armPxRGsSeg { + scope = authzPxRGScope + } + + if len(p) < scope+3 || len(p) > scope+4 || p[0] != armPxSubsSeg || p[scope] != armPxProvSeg || + p[scope+1] != authzPxNS { + return "", "", false + } + + name := "" + if len(p) == scope+4 { + name = p[scope+3] + } + + return p[scope+2], name, true +} + +func authzProxyAllowed(method string, p []string) bool { + typ, name, ok := authzSplit(p) + if !ok { + return false + } + + switch { + case method == http.MethodGet: + return typ == authzPxRoleAssign || typ == authzPxLocks || slices.Contains(authzPxReadOnly, typ) + case name == "": + return false + case typ == authzPxRoleAssign: + return authzPxGUIDName(name) + case typ == authzPxLocks: + return authzPxLockName.MatchString(name) + } + + return false +} + +func authzPxGUIDName(n string) bool { return graphPxGUID.MatchString(n) } + +func authzProxyBody(method string, p []string, in []byte) ([]byte, bool) { + if method != http.MethodPut { + return in, true + } + + typ, _, _ := authzSplit(p) + + var props map[string]any + + switch typ { + case authzPxRoleAssign: + props = authzAssignmentProps(in) + case authzPxLocks: + props = authzLockProps(in) + } + + if props == nil { + return nil, false + } + + b, err := json.Marshal(map[string]any{"properties": props}) + + return b, err == nil +} + +type authzPropsIn struct { + Properties map[string]any `json:"properties"` +} + +func (r authzPropsIn) text(key string) string { + v, _ := r.Properties[key].(string) + + return v +} + +func authzAssignmentProps(in []byte) map[string]any { + var req authzPropsIn + if json.Unmarshal(in, &req) != nil { + return nil + } + + def, pid, typ := req.text("roleDefinitionId"), req.text("principalId"), req.text("principalType") + + if !authzPxRoleDef.MatchString(strings.ToLower(def)) || !graphPxGUID.MatchString(strings.ToLower(pid)) { + return nil + } + + out := map[string]any{"roleDefinitionId": def, "principalId": pid} + + if typ != "" { + if !slices.Contains(authzPxTypes, typ) { + return nil + } + + out["principalType"] = typ + } + + return out +} + +func authzLockProps(in []byte) map[string]any { + var req authzPropsIn + if json.Unmarshal(in, &req) != nil { + return nil + } + + level, notes := req.text("level"), req.text("notes") + if !slices.Contains(authzPxLevels, level) || len(notes) > authzPxMaxNotes { + return nil + } + + out := map[string]any{"level": level} + if notes != "" { + out["notes"] = notes + } + + return out +} diff --git a/dashboard/azureentra_proxy_test.go b/dashboard/azureentra_proxy_test.go new file mode 100644 index 0000000000..a22e5727a3 --- /dev/null +++ b/dashboard/azureentra_proxy_test.go @@ -0,0 +1,362 @@ +package dashboard_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/pkgs/testclient" + "github.com/blackbirdworks/gopherstack/services/azurearm" + "github.com/blackbirdworks/gopherstack/services/azuregraph" +) + +const ( + entraGraph = "/dashboard/api/v1/azure/graph" + entraAuthz = "/dashboard/api/v1/azure/authz" + entraARM = "/dashboard/api/v1/azure/arm" + entraSub = "/subscriptions/" + armSub + entraRG = entraSub + "/resourceGroups/entra" + entraAPI = "?api-version=2022-04-01" + entraAuth = "/providers/Microsoft.Authorization/" + entraGUID1 = "11111111-1111-1111-1111-111111111111" + entraGUID2 = "22222222-2222-2222-2222-222222222222" + entraRole = "/subscriptions/" + armSub + + "/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7" +) + +type entraEnv struct { + client *http.Client + url string +} + +func newEntraEnv(t *testing.T) *entraEnv { + t.Helper() + + armSvc, err := (&azurearm.Provider{}).Init(&service.AppContext{}) + require.NoError(t, err) + + armH, ok := armSvc.(*azurearm.Handler) + require.True(t, ok) + t.Cleanup(func() { armH.Shutdown(context.Background()) }) + + gs := azuregraph.DefaultSettings() + graphH := azuregraph.NewHandler(azuregraph.NewBackend(gs), gs) + + e := echo.New() + e.Any("/*", dashboard.NewHandler(dashboard.Config{AzureARMOps: armH, AzureGraphOps: graphH}).Handler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + env := &entraEnv{url: srv.URL, client: testclient.New(t)} + + code, _ := env.do(t, http.MethodPut, entraARM+entraRG+"?api-version=2021-04-01", `{"location":"eastus"}`) + require.Less(t, code, http.StatusBadRequest) + + return env +} + +func (e *entraEnv) do(t *testing.T, method, path, body string) (int, string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, e.url+path, strings.NewReader(body)) + require.NoError(t, err) + + req.Header.Set("Content-Type", "application/json") + + resp, err := e.client.Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, string(b) +} + +func (e *entraEnv) create(t *testing.T, path, body string) map[string]any { + t.Helper() + + code, out := e.do(t, http.MethodPost, path, body) + require.Less(t, code, http.StatusBadRequest, out) + + var m map[string]any + require.NoError(t, json.Unmarshal([]byte(out), &m)) + + return m +} + +func TestAzureGraphProxyAllowList(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + app := entraGraph + "/applications/" + entraGUID1 + + tests := []struct { + name, method, path, body string + }{ + {"delete user", http.MethodDelete, entraGraph + "/users/" + entraGUID1, ""}, + {"delete group", http.MethodDelete, entraGraph + "/groups/" + entraGUID1, ""}, + {"delete sp", http.MethodDelete, entraGraph + "/servicePrincipals/" + entraGUID1, ""}, + {"patch app", http.MethodPatch, app, `{}`}, + {"put app", http.MethodPut, app, `{}`}, + {"add key", http.MethodPost, app + "/addKey", `{}`}, + {"non guid id", http.MethodGet, entraGraph + "/applications/not-a-guid", ""}, + {"me", http.MethodGet, entraGraph + "/me", ""}, + {"grants", http.MethodGet, entraGraph + "/oauth2PermissionGrants", ""}, + {"organization", http.MethodGet, entraGraph + "/organization", ""}, + {"traversal", http.MethodGet, entraGraph + "/applications/%2e%2e/users", ""}, + {"slash", http.MethodGet, entraGraph + "/applications/" + entraGUID1 + "%2Fowners", ""}, + {"sp password", http.MethodPost, entraGraph + "/servicePrincipals/" + entraGUID1 + "/addPassword", `{}`}, + {"member ref get", http.MethodGet, entraGraph + "/groups/" + entraGUID1 + "/members/$ref", ""}, + {"remove member bad id", http.MethodDelete, entraGraph + "/groups/" + entraGUID1 + "/members/x/$ref", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, body := env.do(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed, http.StatusNotFound}, code) + assert.NotContains(t, body, "Request_") + }) + } +} + +func TestAzureGraphProxyBadBodies(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + + tests := []struct{ name, path, body string }{ + {"app no name", "/applications", `{}`}, + {"app not json", "/applications", `nope`}, + {"sp bad appid", "/servicePrincipals", `{"appId":"x"}`}, + {"user no password", "/users", `{"displayName":"A","mailNickname":"a",` + + `"userPrincipalName":"a@gopherstack.onmicrosoft.com"}`}, + {"user bad upn", "/users", `{"displayName":"A","mailNickname":"a","userPrincipalName":"a b",` + + `"password":"Xx1!xxxxxx"}`}, + {"group bad nick", "/groups", `{"displayName":"G","mailNickname":"a b"}`}, + {"remove password bad key", "/applications/" + entraGUID1 + "/removePassword", `{"keyId":"x"}`}, + {"member bad id", "/groups/" + entraGUID1 + "/members/$ref", `{"id":"http://evil/x"}`}, + {"getbyids empty", "/directoryObjects/getByIds", `{"ids":[]}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, body := env.do(t, http.MethodPost, entraGraph+tt.path, tt.body) + assert.Equal(t, http.StatusBadRequest, code) + assert.Contains(t, body, "InvalidRequestContent") + }) + } +} + +func TestAzureGraphProxySecretOnce(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + + app := env.create(t, entraGraph+"/applications", + `{"displayName":"web","passwordCredentials":[{"secretText":"smuggled"}],`+ + `"signInAudience":"AzureADMultipleOrgs"}`) + id, _ := app["id"].(string) + require.NotEmpty(t, id) + assert.Equal(t, "AzureADMyOrg", app["signInAudience"]) + + pw := env.create(t, entraGraph+"/applications/"+id+"/addPassword", `{"displayName":"ci","extra":"ignored"}`) + secret, _ := pw["secretText"].(string) + require.Len(t, secret, 40) + assert.NotEmpty(t, pw["hint"]) + + paths := []string{"/applications", "/applications/" + id, "/applications?$select=id,passwordCredentials"} + + for _, path := range paths { + code, body := env.do(t, http.MethodGet, entraGraph+path, "") + require.Equal(t, http.StatusOK, code, body) + assert.NotContains(t, body, secret, path) + assert.NotContains(t, body, "secretText", path) + assert.Contains(t, body, `"hint"`, path) + } + + keyID, _ := pw["keyId"].(string) + code, body := env.do(t, http.MethodPost, entraGraph+"/applications/"+id+"/removePassword", `{"keyId":"`+keyID+`"}`) + require.Equal(t, http.StatusNoContent, code, body) + + _, body = env.do(t, http.MethodGet, entraGraph+"/applications/"+id, "") + assert.NotContains(t, body, keyID) + + code, _ = env.do(t, http.MethodDelete, entraGraph+"/applications/"+id, "") + assert.Equal(t, http.StatusNoContent, code) +} + +func TestAzureGraphProxyDirectory(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + const pass = "Sup3r-Secret-Pw!" + + app := env.create(t, entraGraph+"/applications", `{"displayName":"api"}`) + appID, _ := app["appId"].(string) + sp := env.create(t, entraGraph+"/servicePrincipals", `{"appId":"`+appID+`","accountEnabled":false}`) + spID, _ := sp["id"].(string) + assert.Equal(t, true, sp["accountEnabled"]) + + user := env.create(t, entraGraph+"/users", `{"displayName":"Ann","mailNickname":"ann",`+ + `"userPrincipalName":"ann@gopherstack.onmicrosoft.com","password":"`+pass+`","accountEnabled":false}`) + userID, _ := user["id"].(string) + assert.Equal(t, true, user["accountEnabled"]) + + group := env.create(t, entraGraph+"/groups", `{"displayName":"ops","mailNickname":"ops","mailEnabled":true}`) + groupID, _ := group["id"].(string) + assert.Equal(t, false, group["mailEnabled"]) + + code, body := env.do(t, http.MethodPost, entraGraph+"/groups/"+groupID+"/members/$ref", `{"id":"`+userID+`"}`) + require.Equal(t, http.StatusNoContent, code, body) + + _, body = env.do(t, http.MethodGet, entraGraph+"/groups/"+groupID+"/members", "") + assert.Contains(t, body, "ann@gopherstack.onmicrosoft.com") + + code, body = env.do(t, http.MethodPost, entraGraph+"/directoryObjects/getByIds", + `{"ids":["`+userID+`","`+spID+`"],"types":["application"]}`) + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, "Ann") + + code, body = env.do(t, http.MethodDelete, entraGraph+"/groups/"+groupID+"/members/"+userID+"/$ref", "") + require.Equal(t, http.StatusNoContent, code, body) + + code, body = env.do(t, http.MethodGet, entraGraph+"/servicePrincipals/"+spID+"/appRoleAssignments", "") + require.Equal(t, http.StatusOK, code, body) + + for _, path := range []string{"/users", "/users/" + userID, "/groups", "/servicePrincipals"} { + _, out := env.do(t, http.MethodGet, entraGraph+path, "") + assert.NotContains(t, out, pass, path) + assert.NotContains(t, strings.ToLower(out), "passwordprofile", path) + } + + _, out := env.do(t, http.MethodPost, entraGraph+"/users", `{"displayName":"Bob","mailNickname":"bob",`+ + `"userPrincipalName":"bob@gopherstack.onmicrosoft.com","password":"`+pass+`"}`) + assert.NotContains(t, out, pass) + + code, out = env.do(t, http.MethodPost, entraGraph+"/users", `{"displayName":"Weak","mailNickname":"weak",`+ + `"userPrincipalName":"weak@gopherstack.onmicrosoft.com","password":"short"}`) + assert.Equal(t, http.StatusBadRequest, code) + assert.NotContains(t, out, "short\"") +} + +func TestAzureAuthzProxy(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + assignPath := entraAuthz + entraRG + "/providers/Microsoft.Authorization/roleAssignments/" + entraGUID2 + assignBody := `{"properties":{"roleDefinitionId":"` + entraRole + `","principalId":"` + entraGUID1 + + `","principalType":"User","condition":"true","description":"smuggled","scope":"/"}}` + + code, body := env.do(t, http.MethodPut, assignPath+entraAPI, assignBody) + require.Less(t, code, http.StatusBadRequest, body) + assert.NotContains(t, body, "smuggled") + assert.Contains(t, body, entraRG) + + code, body = env.do(t, http.MethodGet, entraAuthz+entraRG+entraAuth+"roleAssignments"+entraAPI, "") + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, entraGUID2) + + code, body = env.do(t, http.MethodGet, entraAuthz+entraSub+entraAuth+"roleDefinitions"+entraAPI, "") + require.Equal(t, http.StatusOK, code, body) + assert.Contains(t, body, "Contributor") + + code, body = env.do(t, http.MethodGet, entraAuthz+entraSub+entraAuth+"policyDefinitions?api-version=2021-06-01", "") + require.Equal(t, http.StatusOK, code, body) + + lock := entraAuthz + entraRG + "/providers/Microsoft.Authorization/locks/keep?api-version=2020-05-01" + lockBody := `{"properties":{"level":"CanNotDelete","notes":"n","owners":[{"applicationId":"x"}]}}` + code, body = env.do(t, http.MethodPut, lock, lockBody) + require.Less(t, code, http.StatusBadRequest, body) + assert.NotContains(t, body, "applicationId") + + code, body = env.do(t, http.MethodDelete, entraARM+entraRG+"?api-version=2021-04-01", "") + assert.Equal(t, http.StatusConflict, code) + assert.Contains(t, body, "ScopeLocked") + + code, _ = env.do(t, http.MethodDelete, lock, "") + assert.Less(t, code, http.StatusMultipleChoices) + + code, _ = env.do(t, http.MethodDelete, assignPath+entraAPI, "") + assert.Less(t, code, http.StatusMultipleChoices) +} + +func TestAzureAuthzProxyRejects(t *testing.T) { + t.Parallel() + + env := newEntraEnv(t) + loc := entraRG + "/providers/Microsoft.Authorization/" + + tests := []struct { + name, method, path, body string + }{ + {"put role definition", http.MethodPut, entraSub + entraAuth + "roleDefinitions/" + entraGUID1, `{}`}, + {"put policy", http.MethodPut, loc + "policyAssignments/x", `{}`}, + {"delete policy", http.MethodDelete, loc + "policyAssignments/x", ""}, + {"other namespace", http.MethodGet, entraRG + "/providers/Microsoft.Storage/storageAccounts", ""}, + { + "resource scope", + http.MethodGet, + entraRG + "/providers/Microsoft.Compute/virtualMachines/a" + entraAuth + "locks", + "", + }, + {"assignment bad name", http.MethodPut, loc + "roleAssignments/not-a-guid", `{}`}, + { + "assignment bad role", http.MethodPut, loc + "roleAssignments/" + entraGUID2, + `{"properties":{"roleDefinitionId":"/providers/x","principalId":"` + entraGUID1 + `"}}`, + }, + { + "assignment bad principal", http.MethodPut, loc + "roleAssignments/" + entraGUID2, + `{"properties":{"roleDefinitionId":"` + entraRole + `","principalId":"nope"}}`, + }, + {"lock bad level", http.MethodPut, loc + "locks/a", `{"properties":{"level":"Nope"}}`}, + {"lock bad name", http.MethodPut, loc + "locks/a%20b", `{"properties":{"level":"ReadOnly"}}`}, + {"post", http.MethodPost, loc + "locks/a", `{}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + code, _ := env.do(t, tt.method, entraAuthz+tt.path+entraAPI, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, code) + }) + } +} + +func TestEntraProxiesUnavailable(t *testing.T) { + t.Parallel() + + e := echo.New() + e.Any("/*", dashboard.NewHandler(dashboard.Config{}).Handler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + env := &entraEnv{url: srv.URL, client: testclient.New(t)} + + for _, path := range []string{ + entraGraph + "/applications", + entraAuthz + entraSub + "/providers/Microsoft.Authorization/roleDefinitions" + entraAPI, + } { + code, body := env.do(t, http.MethodGet, path, "") + assert.Equal(t, http.StatusServiceUnavailable, code, path) + assert.Contains(t, body, "ServiceUnavailable") + } +} diff --git a/dashboard/azuregraph_proxy.go b/dashboard/azuregraph_proxy.go new file mode 100644 index 0000000000..6201fee901 --- /dev/null +++ b/dashboard/azuregraph_proxy.go @@ -0,0 +1,442 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/labstack/echo/v5" +) + +const ( + azureGraphPrefix = "/dashboard/api/v1/azure/graph" + graphPxMaxBody = 1 << 20 + graphPxMaxName = 120 + graphPxMaxPassword = 256 + graphPxMaxIDs = 100 + graphPxMaxTop = 999 + graphPxGraphHost = "https://graph.microsoft.com/v1.0/directoryObjects/" + graphPxApplication = "applications" + graphPxPrincipals = "serviceprincipals" + graphPxUsers = "users" + graphPxGroups = "groups" + graphPxAddPassword = "addpassword" + graphPxRemovePw = "removepassword" + graphPxMembers = "members" + graphPxRef = "$ref" + graphPxPassword = "password" + graphPxDisplayName = "displayName" + graphPxItem = 2 + graphPxSub = 3 +) + +//nolint:gochecknoglobals // immutable allow-lists and validators +var ( + graphPxRoutes = map[string]bool{ + "GET applications": true, "POST applications": true, "GET applications/*": true, + "DELETE applications/*": true, "GET applications/*/owners": true, + "POST applications/*/addpassword": true, "POST applications/*/removepassword": true, + "GET serviceprincipals": true, "POST serviceprincipals": true, "GET serviceprincipals/*": true, + "GET serviceprincipals/*/approleassignments": true, + "GET users": true, "POST users": true, "GET users/*": true, + "GET groups": true, "POST groups": true, "GET groups/*": true, "GET groups/*/members": true, + "POST groups/*/members/$ref": true, "DELETE groups/*/members/*/$ref": true, + "POST directoryobjects/getbyids": true, + } + graphPxGUID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + graphPxSelectRE = regexp.MustCompile(`^[A-Za-z0-9_,]{1,200}$`) + graphPxTokenRE = regexp.MustCompile(`^[A-Za-z0-9_=.\-]{1,1000}$`) + graphPxNickRE = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`) + graphPxUPNRE = regexp.MustCompile(`^[A-Za-z0-9._'+-]{1,64}@[A-Za-z0-9.-]{1,253}$`) + graphPxSecrets = []string{"secrettext", "passwordprofile", graphPxPassword} +) + +type graphProxyErr struct { + Error graphProxyErrBody `json:"error"` +} + +type graphProxyErrBody struct { + Code string `json:"code"` + Message string `json:"message"` +} + +// registerAzureGraphRoutes serves the Entra ID page by dispatching in-process to the Microsoft Graph handler with a +// server-minted token. Paths and methods are allow-listed, every write body is rebuilt from fixed fields, and +// secret-bearing fields are scrubbed from every response except the one addPassword answer that carries a new secret. +func (h *DashboardHandler) registerAzureGraphRoutes() { + ops := h.config.AzureGraphOps + + var inner *echo.Echo + + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Handler()) + } + + serve := func(c *echo.Context) error { + if inner == nil { + return c.JSON(http.StatusServiceUnavailable, graphProxyErr{ + graphProxyErrBody{errServiceUnavailable, "Microsoft Graph is not enabled"}, + }) + } + + r := c.Request() + rest := strings.TrimPrefix(r.URL.EscapedPath(), azureGraphPrefix) + + segs, ok := graphProxySegments(r.Method, rest) + if !ok { + return c.JSON(http.StatusBadRequest, graphProxyErr{ + graphProxyErrBody{"InvalidRequestUri", "unsupported Microsoft Graph proxy request"}, + }) + } + + in, err := io.ReadAll(http.MaxBytesReader(c.Response(), r.Body, graphPxMaxBody)) + if err != nil { + return graphProxyBadBody(c) + } + + body, ok := graphProxyBody(r.Method, segs, in) + if !ok { + return graphProxyBadBody(c) + } + + token, err := ops.InProcessToken() + if err != nil { + return c.JSON(http.StatusInternalServerError, graphProxyErr{ + graphProxyErrBody{"InternalFailure", "could not authorise the Microsoft Graph call"}, + }) + } + + fwd := graphProxyRequest(r, segs, body, token) + rec := httptest.NewRecorder() + inner.ServeHTTP(rec, fwd) + + return graphProxyRelay(c, rec, graphIsAddPassword(r.Method, segs)) + } + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodDelete} { + h.SubRouter.Add(m, azureGraphPrefix+"/*", serve) + } +} + +func graphProxyBadBody(c *echo.Context) error { + return c.JSON(http.StatusBadRequest, graphProxyErr{ + graphProxyErrBody{"InvalidRequestContent", "invalid Microsoft Graph proxy request body"}, + }) +} + +func graphProxyRequest(r *http.Request, segs []string, body []byte, token string) *http.Request { + fwd := r.Clone(r.Context()) + fwd.URL.Path = "/v1.0/" + strings.Join(segs, "/") + fwd.URL.RawPath = "" + fwd.URL.RawQuery = graphProxyQuery(r.URL.Query()).Encode() + fwd.RequestURI = "" + fwd.Header = http.Header{echo.HeaderAuthorization: {"Bearer " + token}} + fwd.Body, fwd.ContentLength = io.NopCloser(bytes.NewReader(body)), int64(len(body)) + + if len(body) > 0 { + fwd.Header.Set("Content-Type", "application/json") + } + + return fwd +} + +// graphProxySegments validates the proxied path and returns its decoded segments, keeping the case the caller sent. +func graphProxySegments(method, raw string) ([]string, bool) { + parts := strings.Split(strings.Trim(raw, "/"), "/") + + for i, p := range parts { + dec, err := url.PathUnescape(p) + if err != nil || dec == "" || strings.ContainsAny(dec, "/:?#\\") || dec == "." || dec == ".." { + return nil, false + } + + parts[i] = dec + } + + if !graphProxyAllowed(method, graphLower(parts)) { + return nil, false + } + + return parts, true +} + +func graphLower(parts []string) []string { + out := make([]string, len(parts)) + for i, p := range parts { + out[i] = strings.ToLower(p) + } + + return out +} + +// graphProxyAllowed matches the lower-cased path, with every GUID segment folded to "*", against graphPxRoutes. +func graphProxyAllowed(method string, p []string) bool { + shape := make([]string, len(p)) + + for i, seg := range p { + shape[i] = seg + if graphPxGUID.MatchString(seg) { + shape[i] = "*" + } + } + + return graphPxRoutes[method+" "+strings.Join(shape, "/")] +} + +func graphIsAddPassword(method string, segs []string) bool { + return method == http.MethodPost && len(segs) == graphPxSub && strings.EqualFold(segs[2], graphPxAddPassword) +} + +func graphProxyQuery(in url.Values) url.Values { + q := url.Values{} + + if v := in.Get("$select"); graphPxSelectRE.MatchString(v) { + q.Set("$select", v) + } + + if v := in.Get("$skiptoken"); graphPxTokenRE.MatchString(v) { + q.Set("$skiptoken", v) + } + + if n, err := strconv.Atoi(in.Get("$top")); err == nil && n >= 1 && n <= graphPxMaxTop { + q.Set("$top", strconv.Itoa(n)) + } + + return q +} + +// graphProxyBody rebuilds the body of every write from fixed fields; reads and deletes carry no body. +func graphProxyBody(method string, segs []string, in []byte) ([]byte, bool) { + if method != http.MethodPost { + return nil, true + } + + var out any + + var ok bool + + p := graphLower(segs) + + switch { + case len(p) == graphPxItem && p[0] == "directoryobjects": + out, ok = graphIDsBody(in) + case len(p) == 1: + out, ok = graphCreateBody(p[0], in) + case p[2] == graphPxAddPassword: + out, ok = graphAddPasswordBody(in) + case p[2] == graphPxRemovePw: + out, ok = graphKeyIDBody(in) + default: + out, ok = graphMemberBody(in) + } + + if !ok { + return nil, false + } + + b, err := json.Marshal(out) + + return b, err == nil +} + +func graphDecode(in []byte, into any) bool { + return json.Unmarshal(in, into) == nil +} + +func graphText(v string) (string, bool) { + v = strings.TrimSpace(v) + control := strings.ContainsFunc(v, func(r rune) bool { return r < ' ' }) + if v == "" || utf8.RuneCountInString(v) > graphPxMaxName || control { + return "", false + } + + return v, true +} + +func graphCreateBody(coll string, in []byte) (any, bool) { + var req struct { + DisplayName string `json:"displayName"` + AppID string `json:"appId"` + MailNickname string `json:"mailNickname"` + UserPrincipalName string `json:"userPrincipalName"` + Password string `json:"password"` + ForceChange bool `json:"forceChangePasswordNextSignIn"` + } + + if !graphDecode(in, &req) { + return nil, false + } + + switch coll { + case graphPxPrincipals: + if !graphPxGUID.MatchString(strings.ToLower(req.AppID)) { + return nil, false + } + + return map[string]any{"appId": req.AppID}, true + case graphPxApplication: + name, ok := graphText(req.DisplayName) + + return map[string]any{graphPxDisplayName: name}, ok + case graphPxGroups: + return graphGroupBody(req.DisplayName, req.MailNickname) + } + + return graphUserBody(req.DisplayName, req.MailNickname, req.UserPrincipalName, req.Password, req.ForceChange) +} + +func graphGroupBody(displayName, nick string) (any, bool) { + name, ok := graphText(displayName) + if !ok || !graphPxNickRE.MatchString(nick) { + return nil, false + } + + return map[string]any{ + graphPxDisplayName: name, "mailNickname": nick, "mailEnabled": false, "securityEnabled": true, + }, true +} + +func graphUserBody(displayName, nick, upn, password string, force bool) (any, bool) { + name, ok := graphText(displayName) + if !ok || !graphPxNickRE.MatchString(nick) || !graphPxUPNRE.MatchString(upn) || + password == "" || len(password) > graphPxMaxPassword { + return nil, false + } + + return map[string]any{ + "accountEnabled": true, graphPxDisplayName: name, "mailNickname": nick, "userPrincipalName": upn, + "passwordProfile": map[string]any{"password": password, "forceChangePasswordNextSignIn": force}, + }, true +} + +func graphAddPasswordBody(in []byte) (any, bool) { + var req struct { + DisplayName string `json:"displayName"` + EndDateTime string `json:"endDateTime"` + } + + if len(bytes.TrimSpace(in)) > 0 && !graphDecode(in, &req) { + return nil, false + } + + cred := map[string]any{} + + if req.DisplayName != "" { + name, ok := graphText(req.DisplayName) + if !ok { + return nil, false + } + + cred[graphPxDisplayName] = name + } + + if req.EndDateTime != "" { + t, err := time.Parse(time.RFC3339, req.EndDateTime) + if err != nil { + return nil, false + } + + cred["endDateTime"] = t.UTC().Format(time.RFC3339) + } + + return map[string]any{"passwordCredential": cred}, true +} + +func graphKeyIDBody(in []byte) (any, bool) { + var req struct { + KeyID string `json:"keyId"` + } + + if !graphDecode(in, &req) || !graphPxGUID.MatchString(strings.ToLower(req.KeyID)) { + return nil, false + } + + return map[string]any{"keyId": req.KeyID}, true +} + +func graphMemberBody(in []byte) (any, bool) { + var req struct { + ID string `json:"id"` + } + + if !graphDecode(in, &req) || !graphPxGUID.MatchString(strings.ToLower(req.ID)) { + return nil, false + } + + return map[string]any{"@odata.id": graphPxGraphHost + req.ID}, true +} + +func graphIDsBody(in []byte) (any, bool) { + var req struct { + IDs []string `json:"ids"` + } + + if !graphDecode(in, &req) || len(req.IDs) == 0 || len(req.IDs) > graphPxMaxIDs { + return nil, false + } + + for _, id := range req.IDs { + if !graphPxGUID.MatchString(strings.ToLower(id)) { + return nil, false + } + } + + return map[string]any{ + "ids": req.IDs, "types": []string{"user", "group", "servicePrincipal", "application"}, + }, true +} + +// graphProxyRelay copies the Graph answer after removing secret-bearing fields. The addPassword answer is the one +// place a freshly generated secret may reach the browser, and it is marked uncacheable. +func graphProxyRelay(c *echo.Context, rec *httptest.ResponseRecorder, keepSecret bool) error { + body := rec.Body.Bytes() + ok := rec.Code < http.StatusMultipleChoices + + if len(body) > 0 && !keepSecret || !ok { + var doc any + if json.Unmarshal(body, &doc) == nil { + graphScrub(doc) + + if out, err := json.Marshal(doc); err == nil { + body = out + } + } + } + + c.Response().Header().Set("Cache-Control", "no-store") + + if len(body) == 0 { + return c.NoContent(rec.Code) + } + + return c.JSONBlob(rec.Code, body) +} + +func graphScrub(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + graphScrub(x) + } + case map[string]any: + for k, x := range t { + if slices.Contains(graphPxSecrets, strings.ToLower(k)) { + delete(t, k) + + continue + } + + graphScrub(x) + } + } +} diff --git a/dashboard/azurenetwork_body.go b/dashboard/azurenetwork_body.go new file mode 100644 index 0000000000..07a007de8b --- /dev/null +++ b/dashboard/azurenetwork_body.go @@ -0,0 +1,268 @@ +package dashboard + +import ( + "encoding/json" + "math" + "net/http" + "net/netip" + "regexp" + "slices" + "strings" +) + +const ( + netPxMinPriority = 100 + netPxMaxPriority = 4096 + netPxMaxDesc = 140 + netPxMaxList = 50 + netPxVNetPutIdx = 8 + netPxProps = "properties" + netPxNameKey = "name" + netPxProtocolKey = "protocol" +) + +var ( + netPxLocationRE = regexp.MustCompile(`^[a-z0-9]{2,40}$`) + netPxNameRE = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9._-]{0,78}[A-Za-z0-9_])?$`) + netPxPortRE = regexp.MustCompile(`^(\*|\d{1,5}(-\d{1,5})?)$`) + netPxTagRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.]{0,63}$`) + netPxDescRE = regexp.MustCompile(`^[^<>"\\]*$`) + netPxServiceRE = regexp.MustCompile(`^Microsoft\.[A-Za-z]{2,40}/[A-Za-z]{2,40}$`) + netPxAssocRE = regexp.MustCompile( + `(?i)^/subscriptions/[0-9a-f-]{1,40}/resourceGroups/[A-Za-z0-9._()-]{1,90}/providers/Microsoft\.Network/` + + `(networkSecurityGroups|natGateways)/[A-Za-z0-9._-]{1,80}$`) + netPxDirections = []string{"Inbound", "Outbound"} //nolint:gochecknoglobals // lookup + netPxAccesses = []string{"Allow", "Deny"} //nolint:gochecknoglobals // lookup + netPxProtocols = []string{"Tcp", "Udp", "Icmp", "Esp", "Ah", "*"} //nolint:gochecknoglobals // lookup +) + +type netPxObj = map[string]any + +// netProxyBody rebuilds the body of every write from fixed fields; anything the caller adds is dropped. +func netProxyBody(method string, p []string, in []byte) ([]byte, bool) { + if method == http.MethodGet || method == http.MethodDelete { + return in, true + } + + if method == http.MethodPost { + return []byte("{}"), true + } + + var req netPxObj + if json.Unmarshal(in, &req) != nil { + return nil, false + } + + props, _ := req[netPxProps].(netPxObj) + + var out netPxObj + + switch { + case len(p) == netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxVNets: + out = netVNetBody(req, props) + case len(p) > netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxVNets && p[netPxVNetPutIdx] == netPxSubnets: + out = netSubnetBody(props) + case len(p) > netPxVNetPutIdx && p[netPxRGTypeIdx] == netPxNSGs && p[netPxVNetPutIdx] == netPxRules: + out = netRuleBody(props) + } + + if out == nil { + return nil, false + } + + b, err := json.Marshal(out) + + return b, err == nil +} + +func netVNetBody(req, props netPxObj) netPxObj { + loc, _ := req["location"].(string) + space, _ := props["addressSpace"].(netPxObj) + prefixes := netCIDRList(space["addressPrefixes"]) + + if !netPxLocationRE.MatchString(loc) || len(prefixes) == 0 { + return nil + } + + out := netPxObj{"addressSpace": netPxObj{"addressPrefixes": prefixes}} + + if raw, present := props["subnets"]; present { + list, _ := raw.([]any) + subnets := make([]any, 0, len(list)) + + for _, e := range list { + m, _ := e.(netPxObj) + name, _ := m[netPxNameKey].(string) + prefix, _ := propsOfAny(m)["addressPrefix"].(string) + + if !netPxNameRE.MatchString(name) || !netValidCIDR(prefix) { + return nil + } + + subnets = append(subnets, netPxObj{netPxNameKey: name, netPxProps: netPxObj{"addressPrefix": prefix}}) + } + + out["subnets"] = subnets + } + + return netPxObj{"location": loc, netPxProps: out} +} + +func propsOfAny(m netPxObj) netPxObj { + p, _ := m[netPxProps].(netPxObj) + + return p +} + +func netSubnetBody(props netPxObj) netPxObj { + prefix, _ := props["addressPrefix"].(string) + if !netValidCIDR(prefix) { + return nil + } + + out := netPxObj{"addressPrefix": prefix} + + for _, k := range []string{"networkSecurityGroup", "natGateway"} { + ref, present := props[k].(netPxObj) + if !present { + continue + } + + id, _ := ref["id"].(string) + if !netPxAssocRE.MatchString(id) { + return nil + } + + out[k] = netPxObj{"id": id} + } + + if raw, present := props["delegations"]; present { + list, _ := raw.([]any) + dels := make([]any, 0, len(list)) + + for _, e := range list { + m, _ := e.(netPxObj) + name, _ := m[netPxNameKey].(string) + svc, _ := propsOfAny(m)["serviceName"].(string) + + if !netPxNameRE.MatchString(name) || !netPxServiceRE.MatchString(svc) { + return nil + } + + dels = append(dels, netPxObj{netPxNameKey: name, netPxProps: netPxObj{"serviceName": svc}}) + } + + out["delegations"] = dels + } + + return netPxObj{netPxProps: out} +} + +func netRuleBody(props netPxObj) netPxObj { + prio, ok := props["priority"].(float64) + if !ok || prio != math.Trunc(prio) || prio < netPxMinPriority || prio > netPxMaxPriority { + return nil + } + + out := netPxObj{"priority": int(prio)} + + for k, allowed := range map[string][]string{ + "direction": netPxDirections, "access": netPxAccesses, netPxProtocolKey: netPxProtocols, + } { + v, _ := props[k].(string) + if !slices.Contains(allowed, v) { + return nil + } + + out[k] = v + } + + if d, present := props["description"].(string); present { + if len(d) > netPxMaxDesc || !netPxDescRE.MatchString(d) { + return nil + } + + out["description"] = d + } + + for _, f := range []struct { + valid func(string) bool + one, many string + }{ + {netPxPortRE.MatchString, "sourcePortRange", "sourcePortRanges"}, + {netPxPortRE.MatchString, "destinationPortRange", "destinationPortRanges"}, + {netValidPrefix, "sourceAddressPrefix", "sourceAddressPrefixes"}, + {netValidPrefix, "destinationAddressPrefix", "destinationAddressPrefixes"}, + } { + if !netCopyValues(out, props, f.one, f.many, f.valid) { + return nil + } + } + + return netPxObj{netPxProps: out} +} + +// netCopyValues copies exactly one of the singular or plural fields once every value validates. +func netCopyValues(out, props netPxObj, one, many string, valid func(string) bool) bool { + if v, ok := props[one].(string); ok && props[many] == nil { + out[one] = v + + return valid(v) + } + + list, ok := props[many].([]any) + if !ok || len(list) == 0 || len(list) > netPxMaxList || props[one] != nil { + return false + } + + vals := make([]string, 0, len(list)) + + for _, e := range list { + s, isStr := e.(string) + if !isStr || !valid(s) { + return false + } + + vals = append(vals, s) + } + + out[many] = vals + + return true +} + +func netValidPrefix(s string) bool { + return s == "*" || netValidCIDR(s) || netPxTagRE.MatchString(s) || netValidAddr(s) +} + +func netValidAddr(s string) bool { + _, err := netip.ParseAddr(s) + + return err == nil +} + +func netValidCIDR(s string) bool { + p, err := netip.ParsePrefix(s) + + return err == nil && !strings.Contains(s, "%") && p.IsValid() +} + +func netCIDRList(v any) []string { + list, _ := v.([]any) + if len(list) == 0 || len(list) > netPxMaxList { + return nil + } + + out := make([]string, 0, len(list)) + + for _, e := range list { + s, ok := e.(string) + if !ok || !netValidCIDR(s) { + return nil + } + + out = append(out, s) + } + + return out +} diff --git a/dashboard/azurenetwork_proxy.go b/dashboard/azurenetwork_proxy.go new file mode 100644 index 0000000000..e8bf19addf --- /dev/null +++ b/dashboard/azurenetwork_proxy.go @@ -0,0 +1,139 @@ +package dashboard + +import ( + "net/http" + "net/url" + "slices" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + azureNetworkPrefix = "/dashboard/api/v1/azure/network" + netPxNamespace = "microsoft.network" + netPxAPIVersion = "2024-05-01" + netPxVNets = "virtualnetworks" + netPxSubnets = "subnets" + netPxNSGs = "networksecuritygroups" + netPxRules = "securityrules" + netPxLBs = "loadbalancers" + netPxAGWs = "applicationgateways" + netPxPolicies = "firewallpolicies" + netPxRCGs = "rulecollectiongroups" + netPxLocations = "locations" + netPxRGList = 7 + netPxRGTypeIdx = 6 + netPxOpSegs = 8 + netPxOpKindIdx = 6 + netPxOpLocIdx = 4 + netPxActionOnly = 1 + netPxChildItem = 2 +) + +var ( + netPxReadTypes = []string{ //nolint:gochecknoglobals // immutable allow-list + netPxVNets, netPxNSGs, "publicipaddresses", "networkinterfaces", "natgateways", netPxLBs, + "privateendpoints", "privatednszones", netPxAGWs, "azurefirewalls", netPxPolicies, "ipgroups", + "applicationsecuritygroups", + } + netPxAGWActions = []string{aciPxStart, aciPxStop, "backendhealth"} //nolint:gochecknoglobals // lookup + netPxOpKinds = []string{"operations", "operationresults"} //nolint:gochecknoglobals // lookup +) + +// registerAzureNetworkRoutes exposes Microsoft.Network resources by dispatching in-process to the ARM handler. +// Reads cover the whole network surface; writes are limited to virtual networks, subnets, security rules, load +// balancer deletion and application gateway start/stop/backend health. Every write forwards a body rebuilt from +// fixed fields, the api-version is pinned, and async operation headers are relayed as proxy-relative URLs. +func (h *DashboardHandler) registerAzureNetworkRoutes() { + h.registerARMProxy(armProxy{ + name: "Azure Networking", prefix: azureNetworkPrefix, + queryKeys: []string{}, version: netPxAPIVersion, relayOps: true, + methods: []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete}, + allowed: netProxyAllowed, + body: netProxyBody, + }) +} + +func netProxyAllowed(method string, p []string) bool { + switch { + case len(p) == netPxOpSegs && p[2] == armPxProvSeg: + return netProxyOperation(method, p) + case len(p) == netPxRGList: + return netProxyRGList(method, p) + } + + typ, name, rest, ok := armScope(p, netPxNamespace) + if !ok || !slices.Contains(netPxReadTypes, typ) { + return false + } + + if name == "" { + return method == http.MethodGet + } + + switch len(rest) { + case 0: + return netProxyItemOp(method, typ) + case netPxActionOnly: + return (method == http.MethodGet && typ == netPxPolicies && rest[0] == netPxRCGs) || + (method == http.MethodPost && typ == netPxAGWs && slices.Contains(netPxAGWActions, rest[0])) + case netPxChildItem: + return netProxyChildOp(method, typ, rest[0]) + } + + return false +} + +func netProxyOperation(method string, p []string) bool { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[2] == armPxProvSeg && p[3] == netPxNamespace && + p[netPxOpLocIdx] == netPxLocations && slices.Contains(netPxOpKinds, p[netPxOpKindIdx]) +} + +func netProxyRGList(method string, p []string) bool { + return method == http.MethodGet && p[0] == armPxSubsSeg && p[2] == armPxRGsSeg && p[4] == armPxProvSeg && + p[5] == netPxNamespace && slices.Contains(netPxReadTypes, p[netPxRGTypeIdx]) +} + +func netProxyItemOp(method, typ string) bool { + switch method { + case http.MethodGet: + return true + case http.MethodPut: + return typ == netPxVNets + case http.MethodDelete: + return typ == netPxVNets || typ == netPxLBs + } + + return false +} + +func netProxyChildOp(method, typ, kind string) bool { + switch { + case typ == netPxVNets && kind == netPxSubnets, typ == netPxNSGs && kind == netPxRules: + return method == http.MethodGet || method == http.MethodPut || method == http.MethodDelete + case typ == netPxPolicies && kind == netPxRCGs: + return method == http.MethodGet + } + + return false +} + +// netRelayOps exposes the async operation URLs of a response as paths below the proxy prefix. +func netRelayOps(c *echo.Context, hdr http.Header, prefix string) { + for _, k := range []string{"Azure-Asyncoperation", "Location"} { + raw := hdr.Get(k) + if raw == "" { + continue + } + + u, err := url.Parse(raw) + if err != nil || !strings.HasPrefix(strings.ToLower(u.Path), "/subscriptions/") { + continue + } + + u.Scheme, u.Host = "", "" + u.Path = prefix + u.Path + c.Response().Header().Set(k, u.String()) + } +} diff --git a/dashboard/azurenetwork_proxy_test.go b/dashboard/azurenetwork_proxy_test.go new file mode 100644 index 0000000000..19ea5fb787 --- /dev/null +++ b/dashboard/azurenetwork_proxy_test.go @@ -0,0 +1,234 @@ +package dashboard_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" +) + +const ( + netAPI = "?api-version=2024-05-01" + netRoot = "/dashboard/api/v1/azure/network" + netNS = armBase + "Microsoft.Network" + netVNetID = netNS + "/virtualNetworks/hub" + netNSGID = netNS + "/networkSecurityGroups/web" + netPolID = netNS + "/firewallPolicies/pol" + netRGList = "/subscriptions/" + armSub + "/resourceGroups/demo/providers/Microsoft.Network/" + netOps = netRoot + "/subscriptions/" + armSub + "/providers/Microsoft.Network/locations/eastus/" + netAGWs = netRoot + netNS + "/applicationGateways/g" + + netBadVNetPrefix = `{"location":"eastus","properties":{"addressSpace":{"addressPrefixes":["nope"]}}}` + netBadVNetRegion = `{"location":"East US","properties":{"addressSpace":{"addressPrefixes":["10.0.0.0/8"]}}}` + netBadNAT = `{"properties":{"addressPrefix":"10.0.5.0/24","natGateway":{"id":"/subscriptions/x/evil"}}}` + netBadDelegation = `{"properties":{"addressPrefix":"10.0.5.0/24",` + + `"delegations":[{"name":"d","properties":{"serviceName":"nope"}}]}}` +) + +func newNetworkDashboard(t *testing.T) *dashboard.DashboardHandler { + t.Helper() + + rule := `{"name":"ssh","properties":{"priority":100,"direction":"Inbound","access":"Allow","protocol":"Tcp",` + + `"sourcePortRange":"*","destinationPortRange":"22","sourceAddressPrefix":"*","destinationAddressPrefix":"*"}}` + rcg := `{"properties":{"priority":200,"ruleCollections":[{"ruleCollectionType":"FirewallPolicyFilterRuleCollection",` + + `"name":"allow","priority":100,"action":{"type":"Allow"},"rules":[{"ruleType":"NetworkRule","name":"r1",` + + `"ipProtocols":["TCP"],"sourceAddresses":["10.0.0.0/24"],"destinationAddresses":["*"],` + + `"destinationPorts":["443"]}]}]}}` + + return newARMDashboard(t, []struct{ path, body string }{ + {"/subscriptions/" + armSub + "/resourceGroups/demo?api-version=2021-04-01", `{"location":"eastus"}`}, + {netNSGID + netAPI, `{"location":"eastus","properties":{"securityRules":[` + rule + `]}}`}, + {netVNetID + netAPI, `{"location":"eastus","properties":{"addressSpace":{"addressPrefixes":["10.0.0.0/16"]},` + + `"subnets":[{"name":"default","properties":{"addressPrefix":"10.0.0.0/24"}}]}}`}, + {netPolID + netAPI, `{"location":"eastus","properties":{"sku":{"tier":"Standard"}}}`}, + {netPolID + "/ruleCollectionGroups/grp" + netAPI, rcg}, + }) +} + +func TestAzureNetworkProxy(t *testing.T) { + t.Parallel() + + px := netRoot + netVNetID + nsg := netRoot + netNSGID + rules := nsg + "/securityRules/" + rule := func(extra string) string { + return `{"properties":{"priority":300,"direction":"Inbound","access":"Deny","protocol":"Udp",` + + `"sourcePortRange":"*","destinationPortRanges":["53","1000-2000"],"sourceAddressPrefix":"Internet",` + + `"destinationAddressPrefix":"10.0.0.0/24"` + extra + `}}` + } + badRule := func(old, repl string) string { return strings.Replace(rule(""), old, repl, 1) } + sub := netRoot + "/subscriptions/" + armSub + "/providers/Microsoft.Network/" + + tests := []struct { + name string + steps []gcpStep + }{ + {"read", []gcpStep{ + {http.MethodGet, netRoot + netRGList + "virtualNetworks" + netAPI, "", `"10.0.0.0/16"`, 200}, + {http.MethodGet, sub + "networkSecurityGroups" + netAPI, "", `"ssh"`, 200}, + {http.MethodGet, px + netAPI, "", `"name":"default"`, 200}, + {http.MethodGet, px + "/subnets/default" + netAPI, "", `"10.0.0.0/24"`, 200}, + {http.MethodGet, nsg + "/securityRules/ssh" + netAPI, "", `"destinationPortRange":"22"`, 200}, + {http.MethodGet, netRoot + netPolID + "/ruleCollectionGroups" + netAPI, "", `"grp"`, 200}, + {http.MethodGet, netRoot + netPolID + "/ruleCollectionGroups/grp" + netAPI, "", `"r1"`, 200}, + {http.MethodGet, netRoot + netRGList + "applicationGateways" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "azureFirewalls" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "ipGroups" + netAPI, "", `"value":[]`, 200}, + {http.MethodGet, netRoot + netRGList + "loadBalancers" + netAPI, "", `"value":[]`, 200}, + }}, + {"pinned version", []gcpStep{ + {http.MethodGet, px + "?api-version=1999-01-01", "", `"name":"default"`, 200}, + {http.MethodGet, px, "", `"name":"default"`, 200}, + }}, + {"writes", []gcpStep{ + { + http.MethodPut, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, + `{"location":"westus","tags":{"x":"y"},"properties":{"dhcpOptions":{"dnsServers":["8.8.8.8"]},` + + `"addressSpace":{"addressPrefixes":["10.1.0.0/16"],"evil":1},` + + `"subnets":[{"name":"app","properties":{"addressPrefix":"10.1.0.0/24","serviceEndpoints":[{}]}}]}}`, + `"10.1.0.0/16"`, 201, + }, + {http.MethodGet, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", `"name":"app"`, 200}, + { + http.MethodPut, netRoot + netNS + "/virtualNetworks/spoke/subnets/db" + netAPI, + `{"properties":{"addressPrefix":"10.1.1.0/24","networkSecurityGroup":{"id":"` + netNSGID + `"},` + + `"delegations":[{"name":"d","properties":{"serviceName":"Microsoft.Web/serverFarms"}}]}}`, + `Microsoft.Web/serverFarms`, 201, + }, + { + http.MethodDelete, + netRoot + netNS + "/virtualNetworks/spoke/subnets/db" + netAPI, + "", + "", + http.StatusAccepted, + }, + {http.MethodPut, rules + "https" + netAPI, rule(`,"description":"dns"`), `"priority":300`, 201}, + {http.MethodGet, nsg + netAPI, "", `"https"`, 200}, + {http.MethodDelete, rules + "https" + netAPI, "", "", http.StatusAccepted}, + {http.MethodDelete, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", "", http.StatusAccepted}, + {http.MethodGet, netRoot + netNS + "/virtualNetworks/spoke" + netAPI, "", "NotFound", http.StatusNotFound}, + }}, + {"body validation", []gcpStep{ + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":99`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":4097`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"priority":300`, `"priority":300.5`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Inbound"`, `"Sideways"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Deny"`, `"Maybe"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Udp"`, `"Gre"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"53"`, `"http"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, badRule(`"Internet"`, `"bad prefix"`), "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, `{"properties":{"priority":300}}`, "invalid", 400}, + {http.MethodPut, rules + "r" + netAPI, `[]`, "invalid", 400}, + {http.MethodPut, px + netAPI, netBadVNetPrefix, "invalid", 400}, + {http.MethodPut, px + netAPI, netBadVNetRegion, "invalid", 400}, + {http.MethodPut, px + "/subnets/x" + netAPI, netBadNAT, "invalid", 400}, + {http.MethodPut, px + "/subnets/x" + netAPI, netBadDelegation, "invalid", 400}, + }}, + {"rejects", []gcpStep{ + {http.MethodDelete, nsg + netAPI, "", "unsupported", 400}, + {http.MethodPut, nsg + netAPI, `{}`, "unsupported", 400}, + {http.MethodPost, px + netAPI, `{}`, "unsupported", 400}, + {http.MethodPut, netRoot + netNS + "/publicIPAddresses/p" + netAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, netAGWs + netAPI, "", "unsupported", 400}, + {http.MethodPost, netAGWs + "/restart" + netAPI, "", "unsupported", 400}, + {http.MethodPut, netRoot + netPolID + "/ruleCollectionGroups/grp" + netAPI, `{}`, "unsupported", 400}, + {http.MethodDelete, netRoot + netPolID + netAPI, "", "unsupported", 400}, + { + http.MethodGet, + netRoot + armBase + "Microsoft.Compute/virtualMachines/x" + netAPI, + "", + "unsupported", + 400, + }, + {http.MethodGet, netRoot + netNS + "/routeTables" + netAPI, "", "unsupported", 400}, + {http.MethodGet, px + "/../x" + netAPI, "", "unsupported", 400}, + {http.MethodGet, netOps + "foo/1", "", "unsupported", 400}, + {http.MethodPost, netOps + "operations/1", "", "unsupported", 400}, + }}, + {"appgw actions reach the RP", []gcpStep{ + {http.MethodPost, netAGWs + "/stop" + netAPI, "", "NotFound", http.StatusNotFound}, + {http.MethodPost, netAGWs + "/backendHealth" + netAPI, "", "NotFound", http.StatusNotFound}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + runGCPSteps(t, newNetworkDashboard(t), tt.steps) + }) + } + + t.Run("async operations are relayed under the proxy", func(t *testing.T) { + t.Parallel() + + h := newNetworkDashboard(t) + do := func(method, path, body string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + + return rec + } + + del := do(http.MethodDelete, px+netAPI, "") + require.Equal(t, http.StatusAccepted, del.Code) + + op := del.Header().Get("Azure-Asyncoperation") + require.True( + t, + strings.HasPrefix(op, netRoot+"/subscriptions/"+armSub+"/providers/Microsoft.Network/locations/"), + op, + ) + assert.NotContains(t, op, "https://") + + status := do(http.MethodGet, op, "") + assert.Equal(t, http.StatusOK, status.Code) + assert.Contains(t, status.Body.String(), "Succeeded") + + loc := del.Header().Get("Location") + require.True(t, strings.HasPrefix(loc, netRoot+"/subscriptions/"), loc) + assert.Equal(t, http.StatusNoContent, do(http.MethodGet, loc, "").Code) + }) + + t.Run("extra fields are dropped", func(t *testing.T) { + t.Parallel() + + h := newNetworkDashboard(t) + do := func(method, path, body string) string { + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Less(t, rec.Code, http.StatusBadRequest, rec.Body.String()) + + return rec.Body.String() + } + + vnet := netRoot + netNS + "/virtualNetworks/spoke" + netAPI + do( + http.MethodPut, + vnet, + `{"location":"westus","tags":{"marker":"y"},"properties":{"dhcpOptions":{"dnsServers":["8.8.8.8"]},`+ + `"addressSpace":{"addressPrefixes":["10.1.0.0/16"]}}}`, + ) + + got := do(http.MethodGet, vnet, "") + assert.NotContains(t, got, "marker") + assert.NotContains(t, got, "8.8.8.8") + assert.Contains(t, got, "10.1.0.0/16") + }) + + t.Run("unserved", func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, sub+"virtualNetworks"+netAPI, nil) + require.NoError(t, dashboard.NewHandler(dashboard.Config{}).Handler()(echo.NewContext(req, rec))) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + }) +} diff --git a/dashboard/gcp_proxy_common.go b/dashboard/gcp_proxy_common.go index 336efbbd6e..49fae5959d 100644 --- a/dashboard/gcp_proxy_common.go +++ b/dashboard/gcp_proxy_common.go @@ -239,6 +239,7 @@ func (h *DashboardHandler) registerConsolePageRoutes() { h.registerAzureContainerAppsRoutes() h.registerAzureVMRoutes() h.registerAzureAKSRoutes() + h.registerAzureNetworkRoutes() h.registerOCIContainerEngineRoutes() h.registerOCIMySQLRoutes() h.registerOCILoggingRoutes() @@ -249,10 +250,14 @@ func (h *DashboardHandler) registerConsolePageRoutes() { h.registerOCILoadBalancerRoutes() h.registerOCIArtifactsRoutes() h.registerOCIContainerInstancesRoutes() + h.registerOCIDNSRoutes() + h.registerOCIDatabaseRoutes() h.registerGCPIdentityToolkitRoutes() h.registerAzureFlexDBRoutes() h.registerGCPAPIGatewayRoutes() h.registerGCPBatchRoutes() h.registerGCPResourceManagerRoutes() h.registerAzureAPIMRoutes() + h.registerAzureGraphRoutes() + h.registerAzureAuthzRoutes() } diff --git a/dashboard/gcpcompute_body.go b/dashboard/gcpcompute_body.go new file mode 100644 index 0000000000..d6cc52801a --- /dev/null +++ b/dashboard/gcpcompute_body.go @@ -0,0 +1,223 @@ +package dashboard + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "regexp" + "strconv" +) + +const ( + computeMaxLabels = 64 + computeMaxLabelLen = 63 + computeMaxDiskGB = 65536 + computeNICAccessName = "External NAT" + computeNICAccessType = "ONE_TO_ONE_NAT" + computeKeyName = "name" +) + +var ( + errComputeBody = errors.New("invalid request body") + errComputeSnapName = errors.New("snapshot name must be 1-63 lowercase letters, digits or dashes") + errComputeInstFields = errors.New("a valid instance name and machine type are required") + errComputeOneEach = errors.New("exactly one boot disk and one network interface are required") + errComputeImage = errors.New("a boot image (projects//global/images/[family/]) is required") + errComputeDiskSize = errors.New("disk size is out of range") + errComputeNetwork = errors.New("invalid network") + errComputeSubnet = errors.New("invalid subnetwork") + errComputeLabels = errors.New("too many labels") + errComputeLabel = errors.New("invalid label") + + computeNameRE = regexp.MustCompile(`^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeMachineRE = regexp.MustCompile(`^(?:zones/[a-z0-9-]+/machineTypes/)?[a-z0-9][-a-z0-9]{0,62}$`) + computeImageRE = regexp.MustCompile( + `^projects/[a-z][-a-z0-9]{0,61}[a-z0-9]/global/images/(?:family/)?[a-z0-9][-a-z0-9]{0,62}$`, + ) + computeNetworkRE = regexp.MustCompile(`^(?:global/networks/)?[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeSubnetRE = regexp.MustCompile(`^(?:regions/[a-z0-9-]+/subnetworks/)?[a-z]([-a-z0-9]{0,61}[a-z0-9])?$`) + computeLabelRE = regexp.MustCompile(`^[a-z0-9_-]{0,63}$`) +) + +type computeInstanceIn struct { + Labels map[string]string `json:"labels"` + Name string `json:"name"` + MachineType string `json:"machineType"` + Disks []computeDiskIn `json:"disks"` + NetworkInterfaces []computeNICIn `json:"networkInterfaces"` +} + +type computeDiskIn struct { + InitializeParams struct { + SourceImage string `json:"sourceImage"` + DiskSizeGb string `json:"diskSizeGb"` + } `json:"initializeParams"` +} + +type computeNICIn struct { + Network string `json:"network"` + Subnetwork string `json:"subnetwork"` + AccessConfigs []struct { + Type string `json:"type"` + } `json:"accessConfigs"` +} + +// gcpComputeBody returns the body forwarded upstream: nothing for reads and actions, and a body rebuilt from fixed +// fields for creating an instance or a snapshot, so metadata, service accounts and every other field are dropped. +func gcpComputeBody(method string, parts []string, in io.Reader) ([]byte, error) { + if method != http.MethodPost { + return nil, nil + } + + switch { + case len(parts) == computeScopedList && parts[computeScopedColl] == computeInstances: + return rebuildComputeInstance(in, parts[computeScopedList-2]) + case len(parts) == computeScopedVerb && parts[computeScopedVerb-1] == computeSnapshotVerb: + return rebuildComputeSnapshot(in) + } + + return nil, nil +} + +func decodeComputeBody(in io.Reader, out any) error { + if err := json.NewDecoder(in).Decode(out); err != nil { + return errComputeBody + } + + return nil +} + +func rebuildComputeSnapshot(in io.Reader) ([]byte, error) { + var req struct { + Name string `json:"name"` + } + + if err := decodeComputeBody(in, &req); err != nil { + return nil, err + } + + if !computeNameRE.MatchString(req.Name) { + return nil, errComputeSnapName + } + + return json.Marshal(map[string]string{computeKeyName: req.Name}) +} + +func rebuildComputeInstance(in io.Reader, zone string) ([]byte, error) { + var req computeInstanceIn + + if err := decodeComputeBody(in, &req); err != nil { + return nil, err + } + + if !computeNameRE.MatchString(req.Name) || !computeMachineRE.MatchString(req.MachineType) { + return nil, errComputeInstFields + } + + if len(req.Disks) != 1 || len(req.NetworkInterfaces) != 1 { + return nil, errComputeOneEach + } + + disk, err := rebuildComputeDisk(req.Disks[0]) + if err != nil { + return nil, err + } + + nic, err := rebuildComputeNIC(req.NetworkInterfaces[0]) + if err != nil { + return nil, err + } + + labels, err := rebuildComputeLabels(req.Labels) + if err != nil { + return nil, err + } + + machine := req.MachineType + if machine == shortComputeName(machine) { + machine = "zones/" + zone + "/machineTypes/" + machine + } + + out := map[string]any{ + computeKeyName: req.Name, "machineType": machine, + "disks": []any{disk}, "networkInterfaces": []any{nic}, + } + if len(labels) > 0 { + out["labels"] = labels + } + + return json.Marshal(out) +} + +func shortComputeName(s string) string { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '/' { + return s[i+1:] + } + } + + return s +} + +func rebuildComputeDisk(d computeDiskIn) (map[string]any, error) { + p := d.InitializeParams + if !computeImageRE.MatchString(p.SourceImage) { + return nil, errComputeImage + } + + init := map[string]any{"sourceImage": p.SourceImage} + + if p.DiskSizeGb != "" { + n, err := strconv.Atoi(p.DiskSizeGb) + if err != nil || n < 1 || n > computeMaxDiskGB { + return nil, errComputeDiskSize + } + + init["diskSizeGb"] = strconv.Itoa(n) + } + + return map[string]any{"boot": true, "autoDelete": true, "initializeParams": init}, nil +} + +func rebuildComputeNIC(n computeNICIn) (map[string]any, error) { + out := map[string]any{} + + if n.Network != "" { + if !computeNetworkRE.MatchString(n.Network) { + return nil, errComputeNetwork + } + + out["network"] = n.Network + } + + if n.Subnetwork != "" { + if !computeSubnetRE.MatchString(n.Subnetwork) { + return nil, errComputeSubnet + } + + out["subnetwork"] = n.Subnetwork + } + + if len(n.AccessConfigs) > 0 { + out["accessConfigs"] = []any{ + map[string]string{computeKeyName: computeNICAccessName, "type": computeNICAccessType}, + } + } + + return out, nil +} + +func rebuildComputeLabels(in map[string]string) (map[string]string, error) { + if len(in) > computeMaxLabels { + return nil, errComputeLabels + } + + for k, v := range in { + if !computeNameRE.MatchString(k) || !computeLabelRE.MatchString(v) { + return nil, errComputeLabel + } + } + + return in, nil +} diff --git a/dashboard/gcpcompute_proxy.go b/dashboard/gcpcompute_proxy.go index 9a35cb8adb..235e3bac69 100644 --- a/dashboard/gcpcompute_proxy.go +++ b/dashboard/gcpcompute_proxy.go @@ -1,6 +1,8 @@ package dashboard import ( + "bytes" + "io" "net/http" "net/url" "slices" @@ -13,6 +15,7 @@ const ( gcpComputeProxyPrefix = "/dashboard/api/v1/gcp/compute" gcpComputeAPIPrefix = "/compute/v1" gcpComputeMaxBodyBytes = 1 << 20 + gcpComputeRevealParam = "reveal" computeMinSegs = 3 computeScopeIdx = 2 @@ -31,22 +34,42 @@ const ( computeDisks = "disks" computeAddresses = "addresses" computeRouters = "routers" + computeSnapshots = "snapshots" + computeSubnets = "subnetworks" + computeBackendSvcs = "backendServices" + computeURLMaps = "urlMaps" + computeHTTPProxies = "targetHttpProxies" + computeHTTPSProxies = "targetHttpsProxies" + computeFwdRules = "forwardingRules" computeStart = "start" computeStop = "stop" computeReset = "reset" + computeSuspend = "suspend" + computeResume = "resume" + computeSnapshotVerb = "createSnapshot" ) //nolint:gochecknoglobals // read-only lookup tables var ( - computeAggregated = []string{ - computeInstances, computeDisks, "subnetworks", computeAddresses, computeRouters, computeOperations, + computeLB = []string{ + computeBackendSvcs, + computeURLMaps, + computeHTTPProxies, + computeHTTPSProxies, + computeFwdRules, } - computeGlobal = []string{ - "networks", "firewalls", "routes", "snapshots", computeOperations, computeAddresses, - } - computeZonal = []string{computeInstances, computeDisks, computeOperations} - computeRegional = []string{"subnetworks", computeAddresses, computeRouters, computeOperations} - computeInstanceActions = []string{computeStart, computeStop, computeReset} + computeAggregated = append([]string{ + computeInstances, computeDisks, computeSubnets, computeAddresses, computeRouters, computeOperations, + }, computeLB...) + computeGlobal = append([]string{ + "networks", "firewalls", "routes", computeSnapshots, computeOperations, computeAddresses, "images", + "securityPolicies", + }, computeLB...) + computeZonal = []string{computeInstances, computeDisks, computeOperations, "machineTypes"} + computeRegional = append( + []string{computeSubnets, computeAddresses, computeRouters, computeOperations}, + computeLB...) + computeInstanceActions = []string{computeStart, computeStop, computeReset, computeSuspend, computeResume} computeQueryKeys = []string{ "filter", "maxResults", @@ -55,8 +78,8 @@ var ( } ) -// registerGCPComputeRoutes exposes Compute Engine browsing and instance, disk, snapshot and address -// management to the SPA by dispatching in-process to the emulator's handler. +// registerGCPComputeRoutes exposes Compute Engine browsing, instance and snapshot creation and resource management to +// the SPA by dispatching in-process to the emulator's handler. Metadata values never leave it unredacted. func (h *DashboardHandler) registerGCPComputeRoutes() { var inner *echo.Echo @@ -72,7 +95,10 @@ func (h *DashboardHandler) registerGCPComputeRoutes() { r := c.Request() - apiPath, ok := gcpComputeProxyPath(r.Method, strings.TrimPrefix(r.URL.EscapedPath(), gcpComputeProxyPrefix)) + apiPath, parts, ok := gcpComputeProxyPath( + r.Method, + strings.TrimPrefix(r.URL.EscapedPath(), gcpComputeProxyPrefix), + ) if !ok { return cloudJSONFail( c, @@ -82,14 +108,30 @@ func (h *DashboardHandler) registerGCPComputeRoutes() { ) } + body, err := gcpComputeBody(r.Method, parts, http.MaxBytesReader(c.Response(), r.Body, gcpComputeMaxBodyBytes)) + if err != nil { + return cloudJSONFail(c, http.StatusBadRequest, cloudStatusBadArg, err.Error()) + } + fwd := r.Clone(r.Context()) fwd.URL.Path, fwd.URL.RawPath = unescapedPath(apiPath), apiPath fwd.URL.RawQuery = gcpComputeQuery(r.URL.Query()) fwd.RequestURI = "" - fwd.Body = http.MaxBytesReader(c.Response(), r.Body, gcpComputeMaxBodyBytes) - inner.ServeHTTP(c.Response(), fwd) + fwd.Body = io.NopCloser(bytes.NewReader(body)) + fwd.ContentLength = int64(len(body)) + + if r.Method != http.MethodGet { + inner.ServeHTTP(c.Response(), fwd) + + return nil + } + + reveal := "" + if len(parts) == computeScopedItem && parts[computeScopedColl] == computeInstances { + reveal = r.URL.Query().Get(gcpComputeRevealParam) + } - return nil + return serveRedacted(c, inner, fwd, reveal) } for _, method := range []string{http.MethodGet, http.MethodPost, http.MethodDelete} { @@ -110,18 +152,19 @@ func gcpComputeQuery(in url.Values) string { } // gcpComputeProxyPath maps /projects/

/... below the proxy prefix to the emulator's /compute/v1 path. Reads cover -// instances, disks, snapshots, networks, subnetworks, firewalls, routes, routers, addresses and operations; writes are -// limited to instance start/stop/reset and deleting instances, disks, snapshots and addresses. -func gcpComputeProxyPath(method, raw string) (string, bool) { +// the compute, network, load balancing, Cloud Armor and operation collections; writes are limited to instance +// start/stop/reset/suspend/resume, creating an instance or a disk snapshot (bodies rebuilt from fixed fields), and +// deleting instances, disks, snapshots and addresses. +func gcpComputeProxyPath(method, raw string) (string, []string, bool) { parts := strings.Split(strings.TrimPrefix(raw, "/"), "/") if len(parts) < computeMinSegs || parts[0] != gcpRESTProjectsSeg { - return "", false + return "", nil, false } for _, p := range parts { dec, err := url.PathUnescape(p) if err != nil || dec == "" || dec == "." || dec == ".." || strings.ContainsAny(dec, "/:") { - return "", false + return "", nil, false } } @@ -137,10 +180,10 @@ func gcpComputeProxyPath(method, raw string) (string, bool) { } if !allowed { - return "", false + return "", nil, false } - return gcpComputeAPIPrefix + "/" + strings.Join(parts, "/"), true + return gcpComputeAPIPrefix + "/" + strings.Join(parts, "/"), parts, true } func computeReadable(parts []string) bool { @@ -172,8 +215,21 @@ func computeScopedRead(parts, collections []string) bool { } func computeActionable(parts []string) bool { - return len(parts) == computeScopedVerb && parts[computeScopeIdx] == computeScopeZones && - parts[computeScopedColl] == computeInstances && slices.Contains(computeInstanceActions, parts[computeScopedVerb-1]) + if parts[computeScopeIdx] != computeScopeZones { + return false + } + + switch len(parts) { + case computeScopedList: + return parts[computeScopedColl] == computeInstances + case computeScopedVerb: + verb := parts[computeScopedVerb-1] + + return (parts[computeScopedColl] == computeInstances && slices.Contains(computeInstanceActions, verb)) || + (parts[computeScopedColl] == computeDisks && verb == computeSnapshotVerb) + } + + return false } func computeDeletable(parts []string) bool { diff --git a/dashboard/gcpcompute_proxy_test.go b/dashboard/gcpcompute_proxy_test.go index 2365e3a629..644f886f44 100644 --- a/dashboard/gcpcompute_proxy_test.go +++ b/dashboard/gcpcompute_proxy_test.go @@ -41,7 +41,8 @@ func seedComputeHandler(t *testing.T) *gcpcompute.Handler { net = `{"name":"vpc","autoCreateSubnetworks":false}` sub = `{"name":"sub","network":"global/networks/vpc","ipCidrRange":"10.10.0.0/24"}` vm = `{"name":"vm1","machineType":"zones/us-central1-a/machineTypes/e2-medium","labels":{"env":"dev"},` + - `"metadata":{"items":[{"key":"startup-script","value":"echo secret-ish"}]},"disks":[{"boot":true,` + + `"metadata":{"items":[{"key":"startup-script","value":"echo secret-ish"},` + + `{"key":"ssh-keys","value":"alice:ssh-rsa AAAAprivate"}]},"disks":[{"boot":true,` + `"autoDelete":true,"initializeParams":{"sourceImage":"projects/debian-cloud/global/images/family/debian-12",` + `"diskSizeGb":"10"}}],"networkInterfaces":[{"subnetwork":"regions/us-central1/subnetworks/sub",` + `"accessConfigs":[{"name":"External NAT","type":"ONE_TO_ONE_NAT"}]}]}` @@ -99,23 +100,45 @@ func TestGCPComputeProxy(t *testing.T) { {http.MethodGet, root + "/global/snapshots", "", "snapshotList", 200}, {http.MethodGet, root + "/aggregated/operations", "", "operation", 200}, {http.MethodGet, root + "/regions/us-central1/addresses/addr1", "", "addr1", 200}, + {http.MethodGet, zone + "/machineTypes", "", "e2-medium", 200}, + {http.MethodGet, root + "/global/images", "", "imageList", 200}, + {http.MethodGet, root + "/global/securityPolicies", "", "securityPolicyList", 200}, + {http.MethodGet, root + "/global/backendServices", "", "backendServiceList", 200}, + {http.MethodGet, root + "/aggregated/backendServices", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/forwardingRules", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/urlMaps", "", "AggregatedList", 200}, + {http.MethodGet, root + "/aggregated/targetHttpProxies", "", "AggregatedList", 200}, + {http.MethodGet, root + "/global/forwardingRules", "", "forwardingRuleList", 200}, + {http.MethodGet, root + "/regions/us-central1/urlMaps", "", "urlMapList", 200}, + {http.MethodGet, root + "/global/operations", "", "operationList", 200}, + {http.MethodGet, zone + "/operations", "", "operationList", 200}, }}, {"actions", []gcpStep{ {http.MethodPost, vm + "/stop", "", `"operationType":"stop"`, 200}, {http.MethodGet, vm, "", "TERMINATED", 200}, {http.MethodPost, vm + "/start", "", `"operationType":"start"`, 200}, {http.MethodPost, vm + "/reset", "", `"operationType":"reset"`, 200}, + {http.MethodPost, vm + "/suspend", "", `"operationType":"suspend"`, 200}, + {http.MethodPost, vm + "/resume", "", `"operationType":"resume"`, 200}, {http.MethodDelete, root + "/regions/us-central1/addresses/addr1", "", `"operationType":"delete"`, 200}, {http.MethodDelete, vm, "", `"operationType":"delete"`, 200}, {http.MethodGet, vm, "", "notFound", 404}, }}, {"rejects", []gcpStep{ - {http.MethodPost, zone + "/instances", `{"name":"x"}`, "unsupported", 400}, + {http.MethodPost, zone + "/instances", `{"name":"x"}`, "machine type are required", 400}, + {http.MethodPost, zone + "/instances", `not json`, "invalid request body", 400}, + {http.MethodPost, zone + "/disks", `{}`, "unsupported", 400}, + {http.MethodPost, zone + "/disks/vm1/resize", `{}`, "unsupported", 400}, + {http.MethodPost, root + "/global/networks", `{"name":"x"}`, "unsupported", 400}, + {http.MethodPost, root + "/global/firewalls", `{"name":"x"}`, "unsupported", 400}, + {http.MethodDelete, root + "/global/firewalls/x", "", "unsupported", 400}, + {http.MethodGet, root + "/global/instanceTemplates", "", "unsupported", 400}, + {http.MethodGet, root + "/projects", "", "unsupported", 400}, {http.MethodPost, vm + "/setMetadata", "{}", "unsupported", 400}, {http.MethodPost, vm + "/stop%3Fx", "", "unsupported", 400}, {http.MethodPut, vm, "{}", "Method Not Allowed", 405}, {http.MethodDelete, root + "/global/networks/vpc", "", "unsupported", 400}, - {http.MethodGet, root + "/global/images", "", "unsupported", 400}, + {http.MethodGet, root + "/global/sslCertificates", "", "unsupported", 400}, {http.MethodGet, root + "/zones/%2e%2e/instances", "", "unsupported", 400}, {http.MethodGet, root + "/global/networks/vpc/extra", "", "unsupported", 400}, {http.MethodGet, "/dashboard/api/v1/gcp/compute/other/p/zones", "", "unsupported", 400}, @@ -144,3 +167,149 @@ func TestGCPComputeProxyUnserved(t *testing.T) { assert.Equal(t, http.StatusServiceUnavailable, rec.Code) assert.Contains(t, rec.Body.String(), "not enabled") } + +func TestGCPComputeProxyMetadataRedaction(t *testing.T) { + t.Parallel() + + const ( + zone = "/dashboard/api/v1/gcp/compute/projects/p/zones/us-central1-a" + vm = zone + "/instances/vm1" + ) + + get := func(t *testing.T, h *dashboard.DashboardHandler, path string) string { + t.Helper() + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, path, nil) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + return rec.Body.String() + } + + tests := []struct { + name string + path string + wantValue []string + wantAbsent []string + }{ + {"item default", vm, []string{`"startup-script"`, `"valueLength":15`, `"redacted":true`}, + []string{"secret-ish", "AAAAprivate"}}, + {"zonal list", zone + "/instances", []string{`"valueLength":15`}, []string{"secret-ish", "AAAAprivate"}}, + {"aggregated list", "/dashboard/api/v1/gcp/compute/projects/p/aggregated/instances", + []string{`"valueLength":15`}, []string{"secret-ish", "AAAAprivate"}}, + {"reveal startup script", vm + "?reveal=startup-script", []string{"echo secret-ish"}, []string{"AAAAprivate"}}, + {"reveal ssh keys refused", vm + "?reveal=ssh-keys", []string{`"redacted":true`}, []string{"AAAAprivate"}}, + {"reveal ignored on list", zone + "/instances?reveal=startup-script", []string{`"redacted":true`}, + []string{"secret-ish"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + body := get(t, h, tt.path) + + for _, w := range tt.wantValue { + assert.Contains(t, body, w) + } + + for _, a := range tt.wantAbsent { + assert.NotContains(t, body, a) + } + }) + } +} + +func TestGCPComputeProxyCreate(t *testing.T) { + t.Parallel() + + const ( + root = "/dashboard/api/v1/gcp/compute/projects/p" + zone = root + "/zones/us-central1-a" + img = "projects/debian-cloud/global/images/family/debian-12" + good = `{"name":"web","machineType":"e2-small","labels":{"env":"dev"},` + + `"metadata":{"items":[{"key":"startup-script","value":"leak-me"}]},` + + `"serviceAccounts":[{"email":"x@y.z","scopes":["a"]}],` + + `"disks":[{"boot":true,"initializeParams":{"sourceImage":"` + img + `","diskSizeGb":"12"}}],` + + `"networkInterfaces":[{"network":"global/networks/vpc","subnetwork":"regions/us-central1/subnetworks/sub",` + + `"accessConfigs":[{"type":"ONE_TO_ONE_NAT","natIP":"1.2.3.4"}]}]}` + ) + + tests := []struct { + name string + steps []gcpStep + }{ + {"rebuilds body", []gcpStep{ + {http.MethodPost, zone + "/instances", good, `"operationType":"insert"`, 200}, + {http.MethodGet, zone + "/instances/web", "", "zones/us-central1-a/machineTypes/e2-small", 200}, + {http.MethodGet, zone + "/instances/web", "", `"env":"dev"`, 200}, + }}, + { + "snapshot", + []gcpStep{ + { + http.MethodPost, + zone + "/disks/vm1/createSnapshot", + `{"name":"snap1","labels":{"a":"b"},"storageLocations":["x"]}`, + `"operationType":"createSnapshot"`, + 200, + }, + {http.MethodGet, root + "/global/snapshots/snap1", "", `"sourceDisk"`, 200}, + {http.MethodPost, zone + "/disks/vm1/createSnapshot", `{"name":"Bad_Name"}`, "snapshot name", 400}, + {http.MethodPost, zone + "/disks/vm1/createSnapshot", `{"name":"snap1"}`, "already exists", 409}, + {http.MethodDelete, root + "/global/snapshots/snap1", "", `"operationType":"delete"`, 200}, + }, + }, + { + "rejects bad fields", + []gcpStep{ + {http.MethodPost, zone + "/instances", `{"name":"Web","machineType":"e2-small"}`, "required", 400}, + { + http.MethodPost, + zone + "/instances", + `{"name":"web","machineType":"e2-small","disks":[],"networkInterfaces":[]}`, + "exactly one", + 400, + }, + {http.MethodPost, zone + "/instances", `{"name":"web","machineType":"e2-small",` + + `"disks":[{"initializeParams":{"sourceImage":"https://evil/x"}}],"networkInterfaces":[{}]}`, "boot image", 400}, + {http.MethodPost, zone + "/instances", `{"name":"web","machineType":"e2-small",` + + `"disks":[{"initializeParams":{"sourceImage":"` + img + `","diskSizeGb":"0"}}],"networkInterfaces":[{}]}`, + "disk size", 400}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + runGCPSteps(t, h, tt.steps) + }) + } +} + +func TestGCPComputeProxyCreateDropsUnlistedFields(t *testing.T) { + t.Parallel() + + const zone = "/dashboard/api/v1/gcp/compute/projects/p/zones/us-central1-a" + + body := `{"name":"web","machineType":"e2-small","metadata":{"items":[{"key":"startup-script","value":"leak-me"}]},` + + `"serviceAccounts":[{"email":"x@y.z","scopes":["a"]}],"tags":{"items":["t"]},` + + `"disks":[{"initializeParams":{"sourceImage":"projects/debian-cloud/global/images/family/debian-12"}}],` + + `"networkInterfaces":[{"network":"vpc","subnetwork":"sub"}]}` + + h := dashboard.NewHandler(dashboard.Config{GCPComputeOps: seedComputeHandler(t)}) + runGCPSteps(t, h, []gcpStep{{http.MethodPost, zone + "/instances", body, "insert", 200}}) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, zone+"/instances/web?reveal=startup-script", nil) + require.NoError(t, h.Handler()(echo.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), "leak-me") + assert.NotContains(t, rec.Body.String(), "x@y.z") + assert.NotContains(t, rec.Body.String(), `"items":["t"]`) +} diff --git a/dashboard/gcpcompute_redact.go b/dashboard/gcpcompute_redact.go new file mode 100644 index 0000000000..5283f07bae --- /dev/null +++ b/dashboard/gcpcompute_redact.go @@ -0,0 +1,111 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + + "github.com/labstack/echo/v5" +) + +const ( + computeMetaKey = "key" + computeMetaValue = "value" + computeMetaLength = "valueLength" + computeMetaHidden = "redacted" + computeMetaField = "metadata" + computeMetaItems = "items" +) + +// serveRedacted runs the upstream request and hides every metadata value in a successful JSON answer. Startup scripts +// and SSH keys live in metadata, so only an explicit reveal of one key that is not SSH-related keeps its value. +func serveRedacted(c *echo.Context, inner *echo.Echo, fwd *http.Request, reveal string) error { + rec := httptest.NewRecorder() + inner.ServeHTTP(rec, fwd) + + body := rec.Body.Bytes() + if rec.Code == http.StatusOK { + body = redactComputeMetadata(body, reveal) + } + + h := c.Response().Header() + for k, v := range rec.Header() { + if k != "Content-Length" { + h[k] = v + } + } + + h.Set("Content-Length", strconv.Itoa(len(body))) + c.Response().WriteHeader(rec.Code) + + _, err := c.Response().Write(body) + + return err +} + +func computeRevealable(key string) bool { + return key != "" && !strings.Contains(strings.ToLower(key), "ssh") +} + +func redactComputeMetadata(body []byte, reveal string) []byte { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + + var doc any + if err := dec.Decode(&doc); err != nil { + return body + } + + redactWalk(doc, reveal) + + out, err := json.Marshal(doc) + if err != nil { + return body + } + + return out +} + +func redactWalk(v any, reveal string) { + switch t := v.(type) { + case map[string]any: + if meta, ok := t[computeMetaField].(map[string]any); ok { + redactItems(meta, reveal) + } + + for _, child := range t { + redactWalk(child, reveal) + } + case []any: + for _, child := range t { + redactWalk(child, reveal) + } + } +} + +func redactItems(meta map[string]any, reveal string) { + items, ok := meta[computeMetaItems].([]any) + if !ok { + return + } + + for _, it := range items { + m, isMap := it.(map[string]any) + if !isMap { + continue + } + + val, _ := m[computeMetaValue].(string) + m[computeMetaLength] = len(val) + + if key, _ := m[computeMetaKey].(string); key == reveal && computeRevealable(key) { + continue + } + + m[computeMetaValue] = "" + m[computeMetaHidden] = true + } +} diff --git a/dashboard/ocidatabase_proxy.go b/dashboard/ocidatabase_proxy.go new file mode 100644 index 0000000000..4884337e98 --- /dev/null +++ b/dashboard/ocidatabase_proxy.go @@ -0,0 +1,440 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "slices" + "strings" + + "github.com/labstack/echo/v5" + + idbackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +const ( + ociDBPrefix = ociProxyPrefix + "/database" + ociDBAPIPrefix = "/20160918" + ociDBDatabases = "autonomousDatabases" + ociDBBackups = "autonomousDatabaseBackups" + ociDBActions = "actions" + ociDBWallet = "wallet" + ociDBMaxBody = 1 << 14 + ociDBMaxName = 255 + ociDBMaxACL = 64 + ociDBMaxPassword = 256 + ociDBItemSegs = 2 + ociDBSubSegs = 3 + ociDBActionSegs = 4 + ociDBKeyComp = "compartmentId" + ociDBKeyDisplay = "displayName" + ociDBKeyLimit = "limit" + ociDBKeyPage = "page" + ociDBKeyState = "lifecycleState" + ociDBKeyPassword = "password" + ociDBKeyAdminPass = "adminPassword" + ociDBKeyACL = "whitelistedIps" + ociDBKeyCompute = "computeCount" + ociDBKeyStorage = "dataStorageSizeInTBs" + ociDBKeyDBID = "autonomousDatabaseId" + ociDBWalletFile = "Wallet_" +) + +var ( + ociDBIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociDBNameRe = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]{0,29}$`) + ociDBACLRe = regexp.MustCompile(`^[A-Za-z0-9.:/_-]{1,255}$`) + ociDBFileRe = regexp.MustCompile(`[^A-Za-z0-9]`) +) + +// ociDBQueryKeys are the only query parameters each listing forwards; everything else is dropped. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDBQueryKeys = map[string][]string{ + ociDBDatabases: { + ociDBKeyComp, ociDBKeyState, ociDBKeyDisplay, "dbWorkload", "isFreeTier", ociDBKeyLimit, ociDBKeyPage, + ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDBBackups: { + ociDBKeyDBID, ociDBKeyComp, ociDBKeyState, ociDBKeyLimit, ociDBKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, +} + +type ociDBCall struct { + body func(c *echo.Context) ([]byte, bool) + path string + coll string + wallet bool +} + +// registerOCIDatabaseRoutes serves the Autonomous Database page by dispatching in-process to the emulator's control +// plane without request signing. Only the database, action, backup and wallet operations the page uses are +// reachable, each write rebuilt from fixed fields. The admin and wallet passwords travel only in the request body +// and are stripped from every answer; the wallet zip is streamed back as an attachment. +func (h *DashboardHandler) registerOCIDatabaseRoutes() { + var inner *echo.Echo + + ops := h.config.OCIDatabaseOps + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Trusted()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return ociGuard(inner != nil, "OCI Autonomous Database", next) + } + + h.SubRouter.Add(http.MethodGet, ociDBPrefix+"/compartments", guard(func(c *echo.Context) error { + comps := ops.Backend.Compartments() + if !slices.Contains(comps, idbackend.TenancyID) { + comps = slices.Insert(comps, 0, idbackend.TenancyID) + } + + return c.JSON(http.StatusOK, map[string]any{ociCompartmentsKey: comps}) + })) + + h.SubRouter.Add(http.MethodGet, ociDBPrefix+"/engine", guard(func(c *echo.Context) error { + return c.JSON(http.StatusOK, map[string]any{"engine": ops.Backend.EngineMode()}) + })) + + serve := guard(func(c *echo.Context) error { + r := c.Request() + + parts, ok := ociSplitPath(strings.TrimPrefix(r.URL.EscapedPath(), ociDBPrefix+"/api")) + if !ok { + return ociInvalid(c, "unsupported Autonomous Database proxy request") + } + + call, ok := ociDBResolve(r.Method, parts) + if !ok { + return ociInvalid(c, "unsupported Autonomous Database proxy request") + } + + var body []byte + + if call.body != nil { + if body, ok = call.body(c); !ok { + return ociInvalid(c, "invalid Autonomous Database request") + } + } + + query := ociArtQuery(r.URL.Query(), ociDBQueryKeys[call.coll]) + r.Header = http.Header{} + r.URL.RawQuery = query + + if call.wallet { + return ociDBWalletDownload(c, inner, parts[1], body) + } + + rec := ociArtForward(c, inner, call.path, body) + if rec.Code < http.StatusMultipleChoices { + rec.Body = bytes.NewBuffer(ociDBScrubJSON(rec.Body.Bytes())) + } + + return ociRelay(c, rec, "opc-next-page", "etag", "opc-request-id", "opc-work-request-id") + }) + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete} { + h.SubRouter.Add(m, ociDBPrefix+"/api/*", serve) + } +} + +// ociDBResolve maps a proxy request to its emulator call; false means the request is outside the allow-list. +func ociDBResolve(method string, parts []string) (ociDBCall, bool) { + if len(parts) > ociDBActionSegs || (len(parts) >= ociDBItemSegs && !ociDBIDOK(parts[1])) { + return ociDBCall{}, false + } + + var ( + call ociDBCall + ok bool + ) + + switch parts[0] { + case ociDBDatabases: + call, ok = ociDBDatabaseCall(method, parts) + case ociDBBackups: + call, ok = ociDBBackupCall(method, parts) + } + + if !ok { + return ociDBCall{}, false + } + + call.coll = parts[0] + call.path = ociDBAPIPrefix + call.path + + return call, true +} + +func ociDBDatabaseCall(method string, parts []string) (ociDBCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDBCall{path: "/autonomousDatabases"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDBCall{path: "/autonomousDatabases", body: ociDBCreateBody}, true + case len(parts) == ociDBItemSegs: + item := "/autonomousDatabases/" + parts[1] + + switch method { + case http.MethodGet, http.MethodDelete: + return ociDBCall{path: item}, true + case http.MethodPut: + return ociDBCall{path: item, body: ociDBUpdateBody}, true + } + case len(parts) == ociDBSubSegs && parts[2] == ociDBWallet && method == http.MethodPost: + return ociDBCall{ + path: "/autonomousDatabases/" + parts[1] + "/actions/generateWallet", body: ociDBWalletBody, wallet: true, + }, true + case len(parts) == ociDBActionSegs && parts[2] == ociDBActions && method == http.MethodPost && ociDBActionOK(parts[3]): + return ociDBCall{path: "/autonomousDatabases/" + parts[1] + "/actions/" + parts[3]}, true + } + + return ociDBCall{}, false +} + +func ociDBBackupCall(method string, parts []string) (ociDBCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDBCall{path: "/autonomousDatabaseBackups"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDBCall{path: "/autonomousDatabaseBackups", body: ociDBBackupBody}, true + } + + return ociDBCall{}, false +} + +func ociDBIDOK(seg string) bool { return ociDBIDRe.MatchString(unescapedPath(seg)) } + +func ociDBActionOK(a string) bool { return a == aciPxStart || a == aciPxStop || a == aciPxRestart } + +func ociDBACLBad(s string) bool { return !ociDBACLRe.MatchString(s) } + +func ociDBText(s string) bool { return s != "" && len(s) <= ociDBMaxName } + +func ociDBWorkloadOK(w string) bool { return slices.Contains([]string{"OLTP", "DW", "AJD", "APEX"}, w) } + +func ociDBModelOK(m string) bool { return m == "" || m == "ECPU" || m == "OCPU" } + +type ociDBCreateIn struct { + Compute *float64 `json:"computeCount"` + Storage *int `json:"dataStorageSizeInTBs"` + FreeTier *bool `json:"isFreeTier"` + Mtls *bool `json:"isMtlsConnectionRequired"` + Comp string `json:"compartmentId"` + Display string `json:"displayName"` + DBName string `json:"dbName"` + Workload string `json:"dbWorkload"` + Model string `json:"computeModel"` + Password string `json:"adminPassword"` +} + +func (in ociDBCreateIn) valid() bool { + return ociDBText(in.Comp) && len(in.Display) <= ociDBMaxName && ociDBNameRe.MatchString(in.DBName) && + ociDBWorkloadOK(in.Workload) && ociDBModelOK(in.Model) && in.Password != "" && + len(in.Password) <= ociDBMaxPassword +} + +func (in ociDBCreateIn) build() map[string]any { + out := map[string]any{ + ociDBKeyComp: in.Comp, "dbName": in.DBName, "dbWorkload": in.Workload, ociDBKeyAdminPass: in.Password, + } + + if in.Display != "" { + out[ociDBKeyDisplay] = in.Display + } + + if in.Model != "" { + out["computeModel"] = in.Model + } + + if in.Compute != nil { + out[ociDBKeyCompute] = *in.Compute + } + + if in.Storage != nil { + out[ociDBKeyStorage] = *in.Storage + } + + if in.FreeTier != nil { + out["isFreeTier"] = *in.FreeTier + } + + if in.Mtls != nil { + out["isMtlsConnectionRequired"] = *in.Mtls + } + + return out +} + +func ociDBCreateBody(c *echo.Context) ([]byte, bool) { + var in ociDBCreateIn + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || !in.valid() { + return nil, false + } + + return ociArtMarshal(in.build()) +} + +func ociDBUpdateBody(c *echo.Context) ([]byte, bool) { + var in struct { + Display *string `json:"displayName"` + ACL *[]string `json:"whitelistedIps"` + ACLOn *bool `json:"isAccessControlEnabled"` + Compute *float64 `json:"computeCount"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) { + return nil, false + } + + out := map[string]any{} + + if in.Display != nil { + if !ociDBText(*in.Display) { + return nil, false + } + + out[ociDBKeyDisplay] = *in.Display + } + + if in.ACL != nil { + if len(*in.ACL) > ociDBMaxACL || slices.ContainsFunc(*in.ACL, ociDBACLBad) { + return nil, false + } + + out[ociDBKeyACL] = *in.ACL + } + + if in.ACLOn != nil { + out["isAccessControlEnabled"] = *in.ACLOn + } + + if in.Compute != nil { + out[ociDBKeyCompute] = *in.Compute + } + + if len(out) == 0 { + return nil, false + } + + return ociArtMarshal(out) +} + +func ociDBBackupBody(c *echo.Context) ([]byte, bool) { + var in struct { + DBID string `json:"autonomousDatabaseId"` + Display string `json:"displayName"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || !ociDBIDRe.MatchString(in.DBID) || len(in.Display) > ociDBMaxName { + return nil, false + } + + out := map[string]any{ociDBKeyDBID: in.DBID} + if in.Display != "" { + out[ociDBKeyDisplay] = in.Display + } + + return ociArtMarshal(out) +} + +func ociDBWalletBody(c *echo.Context) ([]byte, bool) { + var in struct { + Password string `json:"password"` + } + + if !ociDecodeBodyN(c, &in, ociDBMaxBody) || in.Password == "" || len(in.Password) > ociDBMaxPassword { + return nil, false + } + + return ociArtMarshal(map[string]any{ociDBKeyPassword: in.Password, "generateType": "ALL"}) +} + +// ociDBScrubJSON removes password fields from any answer. +func ociDBScrubJSON(raw []byte) []byte { + if len(bytes.TrimSpace(raw)) == 0 { + return raw + } + + var doc any + if json.Unmarshal(raw, &doc) != nil { + return []byte("{}") + } + + ociDBStrip(doc) + + out, err := json.Marshal(doc) + if err != nil { + return []byte("{}") + } + + return out +} + +func ociDBStrip(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + ociDBStrip(x) + } + case map[string]any: + delete(t, ociDBKeyPassword) + delete(t, ociDBKeyAdminPass) + + for _, x := range t { + ociDBStrip(x) + } + } +} + +// ociDBWalletDownload answers GenerateAutonomousDatabaseWallet as a zip attachment. The wallet password is only +// ever in the forwarded body; failures keep the emulator's JSON error envelope. +func ociDBWalletDownload(c *echo.Context, inner *echo.Echo, id string, body []byte) error { + c.Request().Method = http.MethodGet + + meta := ociArtForward(c, inner, ociDBAPIPrefix+"/autonomousDatabases/"+id, nil) + if meta.Code != http.StatusOK { + return ociRelay(c, meta, "opc-request-id") + } + + var db struct { + DBName string `json:"dbName"` + } + + _ = json.Unmarshal(meta.Body.Bytes(), &db) + + c.Request().Method = http.MethodPost + + rec := ociArtForward(c, inner, ociDBAPIPrefix+"/autonomousDatabases/"+id+"/actions/generateWallet", body) + if rec.Code != http.StatusOK { + return ociRelay(c, ociDBScrubRec(rec), "opc-request-id") + } + + h := c.Response().Header() + h.Set("Content-Disposition", `attachment; filename="`+ociDBWalletName(db.DBName)+`"`) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Content-Security-Policy", "sandbox; default-src 'none'") + h.Set("Cache-Control", "no-store") + + return c.Blob(http.StatusOK, "application/zip", rec.Body.Bytes()) +} + +func ociDBScrubRec(rec *httptest.ResponseRecorder) *httptest.ResponseRecorder { + rec.Body = bytes.NewBuffer(ociDBScrubJSON(rec.Body.Bytes())) + + return rec +} + +// ociDBWalletName builds the download file name from the database name, reduced to letters and digits. +func ociDBWalletName(dbName string) string { + name := ociDBFileRe.ReplaceAllString(dbName, "") + if name == "" { + name = "database" + } + + return ociDBWalletFile + name + ".zip" +} diff --git a/dashboard/ocidatabase_proxy_test.go b/dashboard/ocidatabase_proxy_test.go new file mode 100644 index 0000000000..99e2f5011c --- /dev/null +++ b/dashboard/ocidatabase_proxy_test.go @@ -0,0 +1,327 @@ +package dashboard_test + +import ( + "archive/zip" + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" +) + +const ( + dbComp = "ocid1.compartment.oc1..dbcomp" + dbBase = "/dashboard/api/v1/oci/database" + dbAPI = dbBase + "/api" + dbAdminPass = "Sup3rSecretPw#Value" + dbWalletPass = "Sup3rSecretWallet#1" +) + +type dbFixture struct { + h *dashboard.DashboardHandler + ops *ocidatabase.Handler +} + +func newDBDashboard(t *testing.T) dbFixture { + t.Helper() + + be := ocidatabase.NewInMemoryBackend() + be.SetTransition(0) + t.Cleanup(be.Close) + + ops := ocidatabase.NewHandler(be) + + return dbFixture{h: dashboard.NewHandler(dashboard.Config{OCIDatabaseOps: ops}), ops: ops} +} + +func (f dbFixture) direct(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + e := echo.New() + e.Any("/*", f.ops.Trusted()) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + e.ServeHTTP(rec, req) + + return rec +} + +func (f dbFixture) proxy(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, method, path, body) +} + +func (f dbFixture) settle(t *testing.T, id, state string) { + t.Helper() + + require.Eventually(t, func() bool { + f.ops.Backend.Sweep() + + rec := f.direct(t, http.MethodGet, "/20160918/autonomousDatabases/"+id, "") + + var got struct { + State string `json:"lifecycleState"` + } + + return json.Unmarshal(rec.Body.Bytes(), &got) == nil && got.State == state + }, 10*time.Second, 5*time.Millisecond) +} + +func (f dbFixture) create(t *testing.T, extra string) string { + t.Helper() + + rec := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","displayName":"orders","dbName":"ORDERS","dbWorkload":"OLTP", + "adminPassword":"`+dbAdminPass+`","computeModel":"ECPU","computeCount":2,"dataStorageSizeInTBs":1`+extra+`}`) + require.Equal(t, http.StatusAccepted, rec.Code, rec.Body.String()) + + var db struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &db)) + f.settle(t, db.ID, "AVAILABLE") + + return db.ID +} + +func TestOCIDatabaseProxyLifecycle(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + item := dbAPI + "/autonomousDatabases/" + id + + list := f.proxy(t, http.MethodGet, dbAPI+"/autonomousDatabases?compartmentId="+dbComp, "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Contains(t, list.Body.String(), `"dbName":"ORDERS"`) + + for _, step := range []struct{ action, state string }{ + {"stop", "STOPPED"}, {"start", "AVAILABLE"}, {"restart", "AVAILABLE"}, + } { + got := f.proxy(t, http.MethodPost, item+"/actions/"+step.action, "") + require.Less(t, got.Code, http.StatusMultipleChoices, "%s: %s", step.action, got.Body.String()) + f.settle(t, id, step.state) + } + + upd := f.proxy(t, http.MethodPut, item, + `{"displayName":"orders-renamed","whitelistedIps":["203.0.113.0/24"],"isAccessControlEnabled":true, + "computeCount":4,"isFreeTier":true,"adminPassword":"Different#Passw0rd1"}`) + require.Less(t, upd.Code, http.StatusMultipleChoices, upd.Body.String()) + f.settle(t, id, "AVAILABLE") + + got := f.proxy(t, http.MethodGet, item, "") + require.Equal(t, http.StatusOK, got.Code) + assert.Contains(t, got.Body.String(), `"displayName":"orders-renamed"`) + assert.Contains(t, got.Body.String(), `"203.0.113.0/24"`) + assert.Contains(t, got.Body.String(), `"computeCount":4`) + assert.Contains(t, got.Body.String(), `"isFreeTier":false`, "fields outside the allow-list are dropped") + + const longTerm = `,"isLongTermBackup":true,"retentionPeriodInDays":365}` + + bk := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabaseBackups", + `{"autonomousDatabaseId":"`+id+`","displayName":"manual-1"`+longTerm) + require.Equal(t, http.StatusAccepted, bk.Code, bk.Body.String()) + assert.Contains(t, bk.Body.String(), `"displayName":"manual-1"`) + assert.NotContains(t, bk.Body.String(), "LONGTERM") + + query := "?compartmentId=" + dbComp + "&autonomousDatabaseId=" + id + bks := f.proxy(t, http.MethodGet, dbAPI+"/autonomousDatabaseBackups"+query, "") + require.Equal(t, http.StatusOK, bks.Code, bks.Body.String()) + assert.Contains(t, bks.Body.String(), `"manual-1"`) + + del := f.proxy(t, http.MethodDelete, item, "") + assert.Less(t, del.Code, http.StatusMultipleChoices, del.Body.String()) +} + +func TestOCIDatabaseProxyPasswordsNeverEchoed(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + item := dbAPI + "/autonomousDatabases/" + id + + paths := []struct{ method, path, body string }{ + {http.MethodGet, item, ""}, + {http.MethodGet, dbAPI + "/autonomousDatabases?compartmentId=" + dbComp, ""}, + {http.MethodPost, item + "/actions/stop", ""}, + } + + for _, p := range paths { + got := f.proxy(t, p.method, p.path, p.body) + assert.NotContains(t, got.Body.String(), dbAdminPass, "%s %s", p.method, p.path) + assert.NotContains(t, got.Body.String(), "adminPassword", "%s %s", p.method, p.path) + } + + create := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","dbName":"SECOND","dbWorkload":"DW","adminPassword":"`+dbAdminPass+`", + "computeCount":2,"dataStorageSizeInTBs":1}`) + require.Equal(t, http.StatusAccepted, create.Code, create.Body.String()) + assert.NotContains(t, create.Body.String(), dbAdminPass) + + rejected := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases", + `{"compartmentId":"`+dbComp+`","dbName":"THIRD","dbWorkload":"OLTP","adminPassword":"weak"}`) + assert.GreaterOrEqual(t, rejected.Code, http.StatusBadRequest) + assert.NotContains(t, rejected.Body.String(), "weak\"") +} + +func TestOCIDatabaseProxyCreateRebuild(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, `,"freeformTags":{"owner":"mallory"},"isDedicated":true,"isDevTier":true, + "secretId":"ocid1.secret","source":"BACKUP_FROM_ID","whitelistedIps":["1.2.3.4"],"subnetId":"ocid1.subnet.x"`) + + got := f.direct(t, http.MethodGet, "/20160918/autonomousDatabases/"+id, "") + require.Equal(t, http.StatusOK, got.Code) + assert.NotContains(t, got.Body.String(), "mallory") + assert.NotContains(t, got.Body.String(), "1.2.3.4") + assert.NotContains(t, got.Body.String(), "ocid1.subnet.x") +} + +func TestOCIDatabaseProxyWalletAttachment(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + + got := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases/"+id+"/wallet", + `{"password":"`+dbWalletPass+`","generateType":"SINGLE","isRegional":true}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Equal(t, "application/zip", got.Header().Get("Content-Type")) + assert.Equal(t, `attachment; filename="Wallet_ORDERS.zip"`, got.Header().Get("Content-Disposition")) + assert.Equal(t, "nosniff", got.Header().Get("X-Content-Type-Options")) + assert.Equal(t, "no-store", got.Header().Get("Cache-Control")) + assert.Contains(t, got.Header().Get("Content-Security-Policy"), "sandbox") + + zr, err := zip.NewReader(bytes.NewReader(got.Body.Bytes()), int64(got.Body.Len())) + require.NoError(t, err) + + names := make([]string, 0, len(zr.File)) + + for _, zf := range zr.File { + names = append(names, zf.Name) + } + + assert.Contains(t, names, "tnsnames.ora") + assert.Contains(t, names, "ewallet.p12") + assert.NotContains(t, got.Header().Get("Content-Disposition"), dbWalletPass) +} + +func TestOCIDatabaseProxyWalletFailures(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + id := f.create(t, "") + wallet := dbAPI + "/autonomousDatabases/" + id + "/wallet" + + weak := f.proxy(t, http.MethodPost, wallet, `{"password":"short1!"}`) + assert.Equal(t, http.StatusBadRequest, weak.Code, weak.Body.String()) + assert.NotContains(t, weak.Body.String(), "short1!") + assert.Empty(t, weak.Header().Get("Content-Disposition")) + + for _, body := range []string{`{}`, `{"password":""}`, `{`, `{"password":"` + strings.Repeat("a1!", 100) + `"}`} { + got := f.proxy(t, http.MethodPost, wallet, body) + assert.Equal(t, http.StatusBadRequest, got.Code, body) + } + + missing := f.proxy(t, http.MethodPost, dbAPI+"/autonomousDatabases/ocid1.nope/wallet", + `{"password":"`+dbWalletPass+`"}`) + assert.Equal(t, http.StatusNotFound, missing.Code, missing.Body.String()) + assert.NotContains(t, missing.Body.String(), dbWalletPass) + + assert.Equal(t, http.StatusBadRequest, f.proxy(t, http.MethodGet, wallet, "").Code) + assert.Equal(t, http.StatusBadRequest, f.proxy(t, http.MethodPut, wallet, `{"shouldRotate":true}`).Code) +} + +func TestOCIDatabaseProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + item := dbAPI + "/autonomousDatabases/ocid1.db" + valid := `{"compartmentId":"` + dbComp + `","dbName":"X","dbWorkload":"OLTP","adminPassword":"` + dbAdminPass + `"}` + + tests := []struct { + name string + method string + path string + body string + }{ + {"restore", http.MethodPost, item + "/actions/restore", `{"latest":true}`}, + {"change compartment", http.MethodPost, item + "/actions/changeCompartment", `{"compartmentId":"x"}`}, + {"generate wallet action", http.MethodPost, item + "/actions/generateWallet", `{"password":"abcdefg1!"}`}, + {"unknown action", http.MethodPost, item + "/actions/terminate", ""}, + {"rotate wallet", http.MethodPut, item + "/wallet", `{"shouldRotate":true}`}, + {"delete backup", http.MethodDelete, dbAPI + "/autonomousDatabaseBackups/b1", ""}, + {"get backup", http.MethodGet, dbAPI + "/autonomousDatabaseBackups/b1", ""}, + {"work requests", http.MethodGet, dbAPI + "/workRequests", ""}, + {"patch database", http.MethodPatch, item, `{}`}, + {"unknown collection", http.MethodGet, dbAPI + "/cloudExadataInfrastructures", ""}, + {"traversal", http.MethodGet, dbAPI + "/autonomousDatabases/%2e%2e", ""}, + {"encoded slash", http.MethodGet, dbAPI + "/autonomousDatabases/a%2Fb", ""}, + {"admin path", http.MethodPost, dbAPI + "/_gopherstack/keys", `{}`}, + {"too deep", http.MethodGet, item + "/a/b/c", ""}, + {"bad db name", http.MethodPost, dbAPI + "/autonomousDatabases", strings.Replace(valid, `"X"`, `"1 bad"`, 1)}, + {"bad workload", http.MethodPost, dbAPI + "/autonomousDatabases", strings.Replace(valid, `"OLTP"`, `"EXA"`, 1)}, + { + "no password", http.MethodPost, dbAPI + "/autonomousDatabases", + `{"compartmentId":"` + dbComp + `","dbName":"X","dbWorkload":"OLTP"}`, + }, + {"empty update", http.MethodPut, item, `{"freeformTags":{"a":"b"}}`}, + {"bad acl", http.MethodPut, item, `{"whitelistedIps":["a b"]}`}, + {"empty display name", http.MethodPut, item, `{"displayName":""}`}, + {"backup without db", http.MethodPost, dbAPI + "/autonomousDatabaseBackups", `{"displayName":"x"}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDatabaseProxyReadRoutes(t *testing.T) { + t.Parallel() + + f := newDBDashboard(t) + f.create(t, "") + + comps := f.proxy(t, http.MethodGet, dbBase+"/compartments", "") + require.Equal(t, http.StatusOK, comps.Code) + assert.Contains(t, comps.Body.String(), dbComp) + + engine := f.proxy(t, http.MethodGet, dbBase+"/engine", "") + require.Equal(t, http.StatusOK, engine.Code) + assert.Contains(t, engine.Body.String(), `"engine":"stub"`) +} + +func TestOCIDatabaseProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, dbBase + "/compartments"}, + {http.MethodGet, dbBase + "/engine"}, + {http.MethodGet, dbAPI + "/autonomousDatabases?compartmentId=" + dbComp}, + {http.MethodPost, dbAPI + "/autonomousDatabases/x/wallet"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/ocidns_proxy.go b/dashboard/ocidns_proxy.go new file mode 100644 index 0000000000..b6751dfafe --- /dev/null +++ b/dashboard/ocidns_proxy.go @@ -0,0 +1,469 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "net" + "net/http" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v5" + mdns "github.com/miekg/dns" + + idbackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +const ( + ociDNSPrefix = ociProxyPrefix + "/dns" + ociDNSAPIPrefix = "/20180115" + ociDNSZones = "zones" + ociDNSZoneItem = "zone" + ociDNSViews = "views" + ociDNSResolvers = "resolvers" + ociDNSEndpoints = "endpoints" + ociDNSRecords = "records" + ociDNSTsigKeys = "tsigKeys" + ociDNSPolicies = "steeringPolicies" + ociDNSMaxBody = 1 << 16 + ociDNSMaxName = 255 + ociDNSMaxRdata = 4096 + ociDNSMaxItems = 50 + ociDNSMaxTTL = 2147483647 + ociDNSMaxFQDN = 253 + ociDNSMaxLabel = 63 + ociDNSItemSegs = 2 + ociDNSSubSegs = 3 + ociDNSQueryWait = 3 * time.Second + ociDNSKeyLimit = "limit" + ociDNSKeyPage = "page" + ociDNSKeyScope = "scope" + ociDNSKeyViewID = "viewId" + ociDNSKeySortBy = "sortBy" + ociDNSKeySortOrd = "sortOrder" + ociDNSKeyName = "name" + ociDNSKeyState = "lifecycleState" + ociDNSKeyDomain = "domain" + ociDNSKeyRtype = "rtype" + ociDNSKeyDisplay = "displayName" + ociDNSKeyComp = "compartmentId" + ociDNSKeySecret = "secret" + ociDNSKeyPort = "port" + ociDNSKeyType = "type" + ociDNSKeyEnabled = "enabled" + ociDNSOpAdd = "ADD" + ociDNSOpRemove = "REMOVE" + ociDNSScopeGlob = "GLOBAL" + ociDNSScopePriv = "PRIVATE" + ociDNSLoopback = "127.0.0.1" +) + +var ( + ociDNSIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,255}$`) + ociDNSDomainRe = regexp.MustCompile(`^[A-Za-z0-9_*.-]{1,255}$`) + ociDNSLabelRe = regexp.MustCompile(`^[A-Za-z0-9_*]([A-Za-z0-9_-]*[A-Za-z0-9_])?$`) +) + +// ociDNSQueryKeys are the only query parameters each listing forwards; everything else is dropped. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDNSQueryKeys = map[string][]string{ + ociDNSZones: { + ociDNSKeyComp, ociDNSKeyScope, ociDNSKeyViewID, ociDNSKeyName, "nameContains", "zoneType", ociDNSKeyState, + ociDNSKeyLimit, ociDNSKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDNSRecords: { + ociDNSKeyScope, ociDNSKeyViewID, ociDNSKeyDomain, "domainContains", ociDNSKeyRtype, "zoneVersion", + ociDNSKeyLimit, ociDNSKeyPage, ociDNSKeySortBy, ociDNSKeySortOrd, + }, + ociDNSViews: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSResolvers: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSEndpoints: {ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSTsigKeys: {ociDNSKeyComp, ociDNSKeyName, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSPolicies: {ociDNSKeyComp, ociDNSKeyDisplay, ociDNSKeyLimit, ociDNSKeyPage}, + ociDNSZoneItem: {ociDNSKeyScope, ociDNSKeyViewID}, +} + +// ociDNSQueryTypes are the record types the Try it panel may ask for. +// +//nolint:gochecknoglobals // immutable allow-list +var ociDNSQueryTypes = []string{"A", "AAAA", "CNAME", "MX", "NS", "TXT", "SOA", "SRV", "PTR", "CAA"} + +type ociDNSCall struct { + body func(c *echo.Context) ([]byte, bool) + path string + coll string + cond bool +} + +// registerOCIDNSRoutes serves the OCI DNS page by dispatching in-process to the emulator's control plane without +// request signing. Only the zone, record, view, resolver, TSIG key and steering policy operations the page uses are +// reachable, each write rebuilt from fixed fields; TSIG secrets are removed from every answer, and the Try it query +// goes to the emulator's own DNS data plane on the loopback address only. +func (h *DashboardHandler) registerOCIDNSRoutes() { + var inner *echo.Echo + + ops := h.config.OCIDNSOps + if ops != nil { + inner = echo.New() + inner.Any("/*", ops.Trusted()) + } + + guard := func(next echo.HandlerFunc) echo.HandlerFunc { + return ociGuard(inner != nil, "OCI DNS", next) + } + + h.SubRouter.Add(http.MethodGet, ociDNSPrefix+"/compartments", guard(func(c *echo.Context) error { + comps := ops.Backend.Compartments() + if !slices.Contains(comps, idbackend.TenancyID) { + comps = slices.Insert(comps, 0, idbackend.TenancyID) + } + + return c.JSON(http.StatusOK, map[string]any{ociCompartmentsKey: comps}) + })) + + h.SubRouter.Add(http.MethodGet, ociDNSPrefix+"/dataplane", guard(func(c *echo.Context) error { + port := ops.DataPlanePort() + + return c.JSON(http.StatusOK, map[string]any{ociDNSKeyEnabled: port > 0, ociDNSKeyPort: port}) + })) + + h.SubRouter.Add(http.MethodPost, ociDNSPrefix+"/query", guard(func(c *echo.Context) error { + return ociDNSTryIt(c, ops.DataPlanePort()) + })) + + serve := guard(func(c *echo.Context) error { return ociDNSServe(c, inner) }) + + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPatch, http.MethodDelete} { + h.SubRouter.Add(m, ociDNSPrefix+"/api/*", serve) + } +} + +// ociDNSServe forwards one allow-listed control-plane call. +func ociDNSServe(c *echo.Context, inner *echo.Echo) error { + r := c.Request() + + parts, ok := ociSplitPath(strings.TrimPrefix(r.URL.EscapedPath(), ociDNSPrefix+"/api")) + if !ok { + return ociInvalid(c, "unsupported OCI DNS proxy request") + } + + call, ok := ociDNSResolve(r.Method, parts) + if !ok { + return ociInvalid(c, "unsupported OCI DNS proxy request") + } + + var body []byte + + if call.body != nil { + if body, ok = call.body(c); !ok { + return ociInvalid(c, "invalid OCI DNS request") + } + } + + ifMatch := "" + if call.cond { + ifMatch = r.Header.Get("If-Match") + if len(ifMatch) > ociDNSMaxName { + return ociInvalid(c, "invalid If-Match header") + } + } + + query := ociArtQuery(r.URL.Query(), ociDNSQueryKeys[call.coll]) + r.Header = http.Header{} + + if ifMatch != "" { + r.Header.Set("If-Match", ifMatch) + } + + r.URL.RawQuery = query + + rec := ociArtForward(c, inner, call.path, body) + if rec.Code < http.StatusMultipleChoices { + rec.Body = bytes.NewBuffer(ociDNSScrub(rec.Body.Bytes())) + } + + return ociRelay(c, rec, "opc-next-page", "etag", "opc-request-id", "opc-total-items") +} + +// ociDNSResolve maps a proxy request to its emulator call; false means the request is outside the allow-list. +func ociDNSResolve(method string, parts []string) (ociDNSCall, bool) { + if len(parts) > ociDNSSubSegs || (len(parts) >= ociDNSItemSegs && !ociDNSIDOK(parts[1])) { + return ociDNSCall{}, false + } + + var ( + call ociDNSCall + ok bool + ) + + switch parts[0] { + case ociDNSZones: + call, ok = ociDNSZoneCall(method, parts) + case ociDNSViews, ociDNSTsigKeys, ociDNSPolicies: + call, ok = ociDNSCall{path: "/" + parts[0]}, method == http.MethodGet && len(parts) == 1 + case ociDNSResolvers: + call, ok = ociDNSResolverCall(method, parts) + } + + if !ok { + return ociDNSCall{}, false + } + + if call.coll == "" { + call.coll = parts[0] + } + + call.path = ociDNSAPIPrefix + call.path + + return call, true +} + +func ociDNSZoneCall(method string, parts []string) (ociDNSCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDNSCall{path: "/zones"}, true + case len(parts) == 1 && method == http.MethodPost: + return ociDNSCall{path: "/zones", body: ociDNSZoneBody}, true + case len(parts) == ociDNSItemSegs && method == http.MethodGet: + return ociDNSCall{path: "/zones/" + parts[1], coll: ociDNSZoneItem}, true + case len(parts) == ociDNSItemSegs && method == http.MethodDelete: + return ociDNSCall{path: "/zones/" + parts[1], coll: ociDNSZoneItem, cond: true}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSRecords && method == http.MethodGet: + return ociDNSCall{path: "/zones/" + parts[1] + "/records", coll: ociDNSRecords}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSRecords && method == http.MethodPatch: + return ociDNSCall{ + path: "/zones/" + parts[1] + "/records", coll: ociDNSRecords, cond: true, body: ociDNSRecordsBody, + }, true + } + + return ociDNSCall{}, false +} + +func ociDNSResolverCall(method string, parts []string) (ociDNSCall, bool) { + switch { + case len(parts) == 1 && method == http.MethodGet: + return ociDNSCall{path: "/resolvers"}, true + case len(parts) == ociDNSSubSegs && parts[2] == ociDNSEndpoints && method == http.MethodGet: + return ociDNSCall{path: "/resolvers/" + parts[1] + "/endpoints", coll: ociDNSEndpoints}, true + } + + return ociDNSCall{}, false +} + +func ociDNSIDOK(seg string) bool { return ociDNSIDRe.MatchString(unescapedPath(seg)) } + +func ociDNSName(s string) bool { return s != "" && len(s) <= ociDNSMaxName } + +func ociDNSZoneBody(c *echo.Context) ([]byte, bool) { + var in struct { + Name string `json:"name"` + Compartment string `json:"compartmentId"` + ZoneType string `json:"zoneType"` + Scope string `json:"scope"` + ViewID string `json:"viewId"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) { + return nil, false + } + + scope := in.Scope + if scope == "" { + scope = ociDNSScopeGlob + } + + if !ociDNSName(in.Name) || !ociDNSName(in.Compartment) || (in.ZoneType != "" && in.ZoneType != "PRIMARY") || + (scope != ociDNSScopeGlob && scope != ociDNSScopePriv) || + (scope == ociDNSScopePriv) != (in.ViewID != "") || len(in.ViewID) > ociDNSMaxName { + return nil, false + } + + out := map[string]any{ + ociDNSKeyName: in.Name, ociDNSKeyComp: in.Compartment, "zoneType": "PRIMARY", ociDNSKeyScope: scope, + } + + if in.ViewID != "" { + out[ociDNSKeyViewID] = in.ViewID + } + + return ociArtMarshal(out) +} + +type ociDNSRecordIn struct { + TTL *int `json:"ttl"` + Operation string `json:"operation"` + Domain string `json:"domain"` + Rtype string `json:"rtype"` + Rdata string `json:"rdata"` +} + +func (in ociDNSRecordIn) build() (map[string]any, bool) { + if (in.Operation != ociDNSOpAdd && in.Operation != ociDNSOpRemove) || !ociDNSDomainRe.MatchString(in.Domain) || + !slices.Contains(ociDNSQueryTypes, in.Rtype) || in.Rdata == "" || len(in.Rdata) > ociDNSMaxRdata || + strings.ContainsFunc(in.Rdata, func(r rune) bool { return r < ' ' }) { + return nil, false + } + + out := map[string]any{ + "operation": in.Operation, ociDNSKeyDomain: in.Domain, ociDNSKeyRtype: in.Rtype, "rdata": in.Rdata, + } + + if in.TTL != nil { + if *in.TTL < 0 || *in.TTL > ociDNSMaxTTL { + return nil, false + } + + out["ttl"] = *in.TTL + } else if in.Operation == ociDNSOpAdd { + return nil, false + } + + return out, true +} + +func ociDNSRecordsBody(c *echo.Context) ([]byte, bool) { + var in struct { + Items []ociDNSRecordIn `json:"items"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) || len(in.Items) == 0 || len(in.Items) > ociDNSMaxItems { + return nil, false + } + + items := make([]map[string]any, 0, len(in.Items)) + + for _, it := range in.Items { + built, ok := it.build() + if !ok { + return nil, false + } + + items = append(items, built) + } + + return ociArtMarshal(map[string]any{"items": items}) +} + +// ociDNSScrub removes TSIG secrets from any answer. +func ociDNSScrub(raw []byte) []byte { + if len(bytes.TrimSpace(raw)) == 0 { + return raw + } + + var doc any + if json.Unmarshal(raw, &doc) != nil { + return []byte("{}") + } + + ociDNSStrip(doc) + + out, err := json.Marshal(doc) + if err != nil { + return []byte("{}") + } + + return out +} + +func ociDNSStrip(v any) { + switch t := v.(type) { + case []any: + for _, x := range t { + ociDNSStrip(x) + } + case map[string]any: + delete(t, ociDNSKeySecret) + + for _, x := range t { + ociDNSStrip(x) + } + } +} + +// ociDNSQName validates a query name: a dotted host name of at most 253 characters with valid labels. +func ociDNSQName(name string) (string, bool) { + n := strings.TrimSuffix(name, ".") + if n == "" || len(n) > ociDNSMaxFQDN { + return "", false + } + + for label := range strings.SplitSeq(n, ".") { + if len(label) > ociDNSMaxLabel || !ociDNSLabelRe.MatchString(label) { + return "", false + } + } + + return mdns.Fqdn(n), true +} + +type ociDNSRR struct { + Name string `json:"name"` + Type string `json:"type"` + Data string `json:"data"` + TTL uint32 `json:"ttl"` +} + +func ociDNSRRs(in []mdns.RR) []ociDNSRR { + out := make([]ociDNSRR, 0, len(in)) + + for _, rr := range in { + hdr := rr.Header() + out = append(out, ociDNSRR{ + Name: hdr.Name, Type: mdns.TypeToString[hdr.Rrtype], TTL: hdr.Ttl, + Data: strings.TrimSpace(strings.TrimPrefix(rr.String(), hdr.String())), + }) + } + + return out +} + +// ociDNSTryIt runs one query against the emulator's DNS data plane. The address is always the loopback host with +// the configured data plane port; nothing in the request selects where the query goes. +func ociDNSTryIt(c *echo.Context, port int) error { + if port <= 0 { + return cloudJSONFail(c, http.StatusServiceUnavailable, "DataPlaneDisabled", + "the OCI DNS data plane is not running (OCI_DNS_PORT_DNS is 0)") + } + + var in struct { + Name string `json:"name"` + Type string `json:"type"` + } + + if !ociDecodeBodyN(c, &in, ociDNSMaxBody) { + return ociInvalid(c, "invalid query request") + } + + fqdn, ok := ociDNSQName(in.Name) + if !ok || !slices.Contains(ociDNSQueryTypes, in.Type) { + return ociInvalid(c, "query name or type is not valid") + } + + msg := new(mdns.Msg) + msg.SetQuestion(fqdn, mdns.StringToType[in.Type]) + + ctx, cancel := context.WithTimeout(c.Request().Context(), ociDNSQueryWait) + defer cancel() + + addr := net.JoinHostPort(ociDNSLoopback, strconv.Itoa(port)) + + resp, _, err := (&mdns.Client{Net: "udp", Timeout: ociDNSQueryWait}).ExchangeContext(ctx, msg, addr) + if err == nil && resp.Truncated { + resp, _, err = (&mdns.Client{Net: "tcp", Timeout: ociDNSQueryWait}).ExchangeContext(ctx, msg, addr) + } + + if err != nil { + return cloudJSONFail(c, http.StatusBadGateway, "QueryFailed", "the DNS data plane did not answer") + } + + return c.JSON(http.StatusOK, map[string]any{ + "rcode": mdns.RcodeToString[resp.Rcode], "authoritative": resp.Authoritative, + "answer": ociDNSRRs(resp.Answer), "authority": ociDNSRRs(resp.Ns), "additional": ociDNSRRs(resp.Extra), + ociDNSKeyName: fqdn, ociDNSKeyType: in.Type, "server": addr, + }) +} diff --git a/dashboard/ocidns_proxy_test.go b/dashboard/ocidns_proxy_test.go new file mode 100644 index 0000000000..99e5bb0a33 --- /dev/null +++ b/dashboard/ocidns_proxy_test.go @@ -0,0 +1,409 @@ +package dashboard_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/dashboard" + "github.com/blackbirdworks/gopherstack/services/ocidns" +) + +const ( + dnsComp = "ocid1.compartment.oc1..dnscomp" + dnsBase = "/dashboard/api/v1/oci/dns" + dnsAPI = dnsBase + "/api" + dnsSecret = "c3VwZXItc2VjcmV0LXRzaWctdmFsdWU=" +) + +type dnsFixture struct { + h *dashboard.DashboardHandler + ops *ocidns.Handler +} + +func newDNSDashboard(t *testing.T, dataPlane bool) dnsFixture { + t.Helper() + + ops := ocidns.NewHandler(ocidns.NewInMemoryBackend()) + ops.Port, ops.DNSPort, ops.DNSEphemeral = 0, 0, dataPlane + + require.NoError(t, ops.StartWorker(t.Context())) + t.Cleanup(func() { ops.Shutdown(context.Background()) }) + + return dnsFixture{h: dashboard.NewHandler(dashboard.Config{OCIDNSOps: ops}), ops: ops} +} + +func (f dnsFixture) direct(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + e := echo.New() + e.Any("/*", f.ops.Trusted()) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + e.ServeHTTP(rec, req) + + return rec +} + +func (f dnsFixture) proxy(t *testing.T, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + + return lbDo(t, f.h, method, path, body) +} + +func (f dnsFixture) zone(t *testing.T, name string) string { + t.Helper() + + rec := f.proxy(t, http.MethodPost, dnsAPI+"/zones", + `{"name":"`+name+`","compartmentId":"`+dnsComp+`","zoneType":"PRIMARY","scope":"GLOBAL"}`) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + + var z struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &z)) + + return z.ID +} + +func (f dnsFixture) addRecord(t *testing.T, zoneID, domain, rtype, rdata string) { + t.Helper() + + rec := f.proxy(t, http.MethodPatch, dnsAPI+"/zones/"+zoneID+"/records", + `{"items":[{"operation":"ADD","domain":"`+domain+`","rtype":"`+rtype+`","rdata":"`+rdata+`","ttl":60}]}`) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) +} + +func TestOCIDNSProxyZoneAndRecordLifecycle(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + rec := f.proxy(t, http.MethodPost, dnsAPI+"/zones", + `{"name":"example.com","compartmentId":"`+dnsComp+`","zoneType":"PRIMARY","scope":"GLOBAL", + "freeformTags":{"owner":"mallory"},"migrationSource":"DYNECT","externalMasters":[{"address":"10.9.9.9"}]}`) + require.Less(t, rec.Code, http.StatusMultipleChoices, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), "mallory") + assert.NotContains(t, rec.Body.String(), "10.9.9.9") + + var zone struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &zone)) + + list := f.proxy(t, http.MethodGet, dnsAPI+"/zones?compartmentId="+dnsComp+"&scope=GLOBAL", "") + require.Equal(t, http.StatusOK, list.Code, list.Body.String()) + assert.Contains(t, list.Body.String(), `"name":"example.com"`) + + records := dnsAPI + "/zones/" + zone.ID + "/records" + f.addRecord(t, zone.ID, "www.example.com", "A", "192.0.2.1") + f.addRecord(t, zone.ID, "app.example.com", "CNAME", "www.example.com.") + + got := f.proxy(t, http.MethodGet, records, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"rdata":"192.0.2.1"`) + assert.Contains(t, got.Body.String(), `"rtype":"CNAME"`) + require.NotEmpty(t, got.Header().Get("ETag"), "etag is relayed for If-Match") + + edit := f.proxy(t, http.MethodPatch, records, `{"items":[ + {"operation":"REMOVE","domain":"www.example.com","rtype":"A","rdata":"192.0.2.1","ttl":60}, + {"operation":"ADD","domain":"www.example.com","rtype":"A","rdata":"192.0.2.2","ttl":120}]}`) + require.Equal(t, http.StatusOK, edit.Code, edit.Body.String()) + + got = f.proxy(t, http.MethodGet, records+"?domain=www.example.com&rtype=A", "") + assert.Contains(t, got.Body.String(), `"rdata":"192.0.2.2"`) + assert.NotContains(t, got.Body.String(), `"rdata":"192.0.2.1"`) + + del := f.proxy(t, http.MethodPatch, records, + `{"items":[{"operation":"REMOVE","domain":"www.example.com","rtype":"A","rdata":"192.0.2.2"}]}`) + require.Equal(t, http.StatusOK, del.Code, del.Body.String()) + + gone := f.proxy(t, http.MethodDelete, dnsAPI+"/zones/"+zone.ID, "") + assert.Less(t, gone.Code, http.StatusMultipleChoices, gone.Body.String()) +} + +func TestOCIDNSProxyEtagConflict(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + id := f.zone(t, "conflict.example") + records := dnsAPI + "/zones/" + id + "/records" + + first := f.proxy(t, http.MethodGet, records, "") + etag := first.Header().Get("ETag") + require.NotEmpty(t, etag) + + body := `{"items":[{"operation":"ADD","domain":"a.conflict.example","rtype":"A","rdata":"192.0.2.1","ttl":60}]}` + + patch := func(ifMatch string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, records, strings.NewReader(body)) + req.Header.Set("If-Match", ifMatch) + req.Header.Set("Authorization", "Bearer should-not-reach-emulator") + require.NoError(t, f.h.Handler()(echo.NewContext(req, rec))) + + return rec + } + + require.Equal(t, http.StatusOK, patch(etag).Code) + + stale := patch(etag) + assert.Equal(t, http.StatusPreconditionFailed, stale.Code, stale.Body.String()) + assert.Contains(t, stale.Body.String(), "NoEtagMatch") +} + +func TestOCIDNSProxyRejectsOutsideAllowList(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + id := f.zone(t, "reject.example") + rec := dnsAPI + "/zones/" + id + "/records" + + tests := []struct { + name string + method string + path string + body string + }{ + {"delete tsig key", http.MethodDelete, dnsAPI + "/tsigKeys/k1", ""}, + {"get one tsig key", http.MethodGet, dnsAPI + "/tsigKeys/k1", ""}, + {"update zone", http.MethodPut, dnsAPI + "/zones/" + id, `{}`}, + {"create resolver endpoint", http.MethodPost, dnsAPI + "/resolvers/r1/endpoints", `{}`}, + {"create view", http.MethodPost, dnsAPI + "/views", `{}`}, + {"zone content", http.MethodGet, dnsAPI + "/zones/" + id + "/content", ""}, + {"replace records", http.MethodPut, rec, `{"items":[]}`}, + {"domain records", http.MethodGet, rec + "/www.reject.example", ""}, + {"attachments", http.MethodGet, dnsAPI + "/steeringPolicyAttachments", ""}, + {"traversal", http.MethodGet, dnsAPI + "/zones/%2e%2e", ""}, + {"encoded slash", http.MethodGet, dnsAPI + "/zones/a%2Fb", ""}, + {"empty api", http.MethodGet, dnsAPI + "/", ""}, + {"unknown collection", http.MethodGet, dnsAPI + "/admin", ""}, + {"admin path", http.MethodPost, dnsAPI + "/_gopherstack/keys", `{}`}, + { + "secondary zone", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","zoneType":"SECONDARY"}`, + }, + { + "private zone without view", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","scope":"PRIVATE"}`, + }, + { + "global zone with view", http.MethodPost, dnsAPI + "/zones", + `{"name":"x.example","compartmentId":"` + dnsComp + `","scope":"GLOBAL","viewId":"v"}`, + }, + {"zone without name", http.MethodPost, dnsAPI + "/zones", `{"compartmentId":"` + dnsComp + `"}`}, + { + "unknown operation", http.MethodPatch, rec, + `{"items":[{"operation":"REQUIRE","domain":"a.reject.example","rtype":"A","rdata":"1.1.1.1","ttl":1}]}`, + }, + { + "bad record type", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a.reject.example","rtype":"ALIAS","rdata":"x","ttl":1}]}`, + }, + { + "add without ttl", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a.reject.example","rtype":"A","rdata":"1.1.1.1"}]}`, + }, + { + "bad domain", http.MethodPatch, rec, + `{"items":[{"operation":"ADD","domain":"a b","rtype":"A","rdata":"1.1.1.1","ttl":1}]}`, + }, + {"empty items", http.MethodPatch, rec, `{"items":[]}`}, + {"malformed body", http.MethodPatch, rec, `{`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, tt.method, tt.path, tt.body) + assert.Contains(t, []int{http.StatusBadRequest, http.StatusMethodNotAllowed}, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDNSProxyTsigSecretStripped(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + created := f.direct(t, http.MethodPost, "/20180115/tsigKeys", + `{"name":"transfer.example","algorithm":"hmac-sha256","secret":"`+dnsSecret+`","compartmentId":"`+dnsComp+`"}`) + require.Less(t, created.Code, http.StatusMultipleChoices, created.Body.String()) + require.Contains(t, created.Body.String(), dnsSecret, "fixture: the emulator returns the secret on create") + + var key struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(created.Body.Bytes(), &key)) + + direct := f.direct(t, http.MethodGet, "/20180115/tsigKeys/"+key.ID, "") + require.Contains(t, direct.Body.String(), dnsSecret, "fixture: the emulator returns the secret on get") + + got := f.proxy(t, http.MethodGet, dnsAPI+"/tsigKeys?compartmentId="+dnsComp, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"name":"transfer.example"`) + assert.Contains(t, got.Body.String(), `"algorithm":"hmac-sha256"`) + assert.NotContains(t, got.Body.String(), dnsSecret) + assert.NotContains(t, got.Body.String(), `"secret"`) + + one := f.proxy(t, http.MethodGet, dnsAPI+"/tsigKeys/"+key.ID, "") + assert.Equal(t, http.StatusBadRequest, one.Code) +} + +func TestOCIDNSProxyReadOnlyLists(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + view := f.direct(t, http.MethodPost, "/20180115/views", `{"compartmentId":"`+dnsComp+`","displayName":"corp-view"}`) + require.Less(t, view.Code, http.StatusMultipleChoices, view.Body.String()) + + policy := f.direct(t, http.MethodPost, "/20180115/steeringPolicies", + `{"compartmentId":"`+dnsComp+`","displayName":"failover","template":"FAILOVER","ttl":30, + "answers":[{"name":"a","rtype":"A","rdata":"192.0.2.9","pool":"p"}], + "rules":[{"ruleType":"FILTER", + "defaultAnswerData":[{"answerCondition":"answer.isDisabled != true","shouldKeep":true}]}]}`) + require.Less(t, policy.Code, http.StatusMultipleChoices, policy.Body.String()) + + tests := []struct { + name string + path string + want string + }{ + {"compartments", dnsBase + "/compartments", dnsComp}, + {"views", dnsAPI + "/views?compartmentId=" + dnsComp, `"displayName":"corp-view"`}, + {"policies", dnsAPI + "/steeringPolicies?compartmentId=" + dnsComp, `"displayName":"failover"`}, + {"resolvers", dnsAPI + "/resolvers?compartmentId=" + dnsComp, `[`}, + {"dataplane", dnsBase + "/dataplane", `"enabled":false`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := f.proxy(t, http.MethodGet, tt.path, "") + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), tt.want) + }) + } +} + +func TestOCIDNSProxyTryIt(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, true) + port := f.ops.DataPlanePort() + require.Positive(t, port) + + id := f.zone(t, "tryit.example") + f.addRecord(t, id, "www.tryit.example", "A", "192.0.2.77") + + dp := f.proxy(t, http.MethodGet, dnsBase+"/dataplane", "") + assert.Contains(t, dp.Body.String(), `"enabled":true`) + + query := func(body string) *httptest.ResponseRecorder { + return f.proxy(t, http.MethodPost, dnsBase+"/query", body) + } + + t.Run("answers from the data plane", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"www.tryit.example","type":"A"}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + + var out map[string]any + + require.NoError(t, json.Unmarshal(got.Body.Bytes(), &out)) + assert.Equal(t, "NOERROR", out["rcode"]) + assert.Equal(t, true, out["authoritative"]) + + answer, ok := out["answer"].([]any) + require.True(t, ok) + require.Len(t, answer, 1) + + rr, ok := answer[0].(map[string]any) + require.True(t, ok) + assert.Equal(t, "A", rr["type"]) + assert.Equal(t, "192.0.2.77", rr["data"]) + assert.InDelta(t, 60, rr["ttl"], 0) + }) + + t.Run("loopback only", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"www.tryit.example.","type":"A","server":"8.8.8.8:53","address":"203.0.113.1","port":53}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"server":"127.0.0.1:`) + assert.NotContains(t, got.Body.String(), "8.8.8.8") + assert.NotContains(t, got.Body.String(), "203.0.113.1") + }) + + t.Run("unknown name is nxdomain", func(t *testing.T) { + t.Parallel() + + got := query(`{"name":"missing.tryit.example","type":"A"}`) + require.Equal(t, http.StatusOK, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), `"rcode":"NXDOMAIN"`) + }) + + invalid := []struct{ name, body string }{ + {"space in name", `{"name":"bad name.example","type":"A"}`}, + {"empty name", `{"name":"","type":"A"}`}, + {"empty label", `{"name":"a..example","type":"A"}`}, + {"long label", `{"name":"` + strings.Repeat("a", 64) + `.example","type":"A"}`}, + {"long name", `{"name":"` + strings.Repeat("abcdefgh.", 30) + `example","type":"A"}`}, + {"zone transfer", `{"name":"tryit.example","type":"AXFR"}`}, + {"any", `{"name":"tryit.example","type":"ANY"}`}, + {"lowercase type", `{"name":"tryit.example","type":"a"}`}, + {"malformed", `{`}, + } + + for _, tt := range invalid { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := query(tt.body) + assert.Equal(t, http.StatusBadRequest, got.Code, got.Body.String()) + }) + } +} + +func TestOCIDNSProxyTryItWithoutDataPlane(t *testing.T) { + t.Parallel() + + f := newDNSDashboard(t, false) + + got := f.proxy(t, http.MethodPost, dnsBase+"/query", `{"name":"www.example.com","type":"A"}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, got.Body.String()) + assert.Contains(t, got.Body.String(), "DataPlaneDisabled") +} + +func TestOCIDNSProxyUnavailableWithoutService(t *testing.T) { + t.Parallel() + + h := dashboard.NewHandler(dashboard.Config{}) + + for _, p := range []struct{ method, path string }{ + {http.MethodGet, dnsBase + "/compartments"}, + {http.MethodGet, dnsBase + "/dataplane"}, + {http.MethodPost, dnsBase + "/query"}, + {http.MethodGet, dnsAPI + "/zones?compartmentId=" + dnsComp}, + {http.MethodPatch, dnsAPI + "/zones/z1/records"}, + } { + got := lbDo(t, h, p.method, p.path, `{}`) + assert.Equal(t, http.StatusServiceUnavailable, got.Code, "%s %s", p.method, p.path) + } +} diff --git a/dashboard/provider.go b/dashboard/provider.go index 1769675206..ca73d1cd80 100644 --- a/dashboard/provider.go +++ b/dashboard/provider.go @@ -23,6 +23,7 @@ import ( "github.com/blackbirdworks/gopherstack/services/azureeventhubs" "github.com/blackbirdworks/gopherstack/services/azurefile" "github.com/blackbirdworks/gopherstack/services/azurefunctions" + "github.com/blackbirdworks/gopherstack/services/azuregraph" "github.com/blackbirdworks/gopherstack/services/azurekeyvault" "github.com/blackbirdworks/gopherstack/services/azuremonitor" "github.com/blackbirdworks/gopherstack/services/azurequeue" @@ -80,6 +81,8 @@ import ( "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + "github.com/blackbirdworks/gopherstack/services/ocidns" "github.com/blackbirdworks/gopherstack/services/ocievents" "github.com/blackbirdworks/gopherstack/services/ocifunctions" "github.com/blackbirdworks/gopherstack/services/ociidentity" @@ -164,6 +167,10 @@ func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { conf.AzureARMOps, _ = getHandler[*azurearm.Handler](ap.GetAzureARMHandler()) } + if gp, hasGraph := ctx.Config.(interface{ GetAzureGraphHandler() service.Registerable }); hasGraph { + conf.AzureGraphOps, _ = getHandler[*azuregraph.Handler](gp.GetAzureGraphHandler()) + } + if bp, hasBlob := ctx.Config.(interface{ GetAzureBlobHandler() service.Registerable }); hasBlob { conf.AzureBlobOps, _ = getHandler[*azureblob.Handler](bp.GetAzureBlobHandler()) } @@ -484,6 +491,14 @@ func linkOCIObservabilityExtras(conf *Config, cfg any) { p.GetOCIContainerInstancesHandler()) } + if p, ok := cfg.(interface{ GetOCIDNSHandler() service.Registerable }); ok { + conf.OCIDNSOps, _ = getHandler[*ocidns.Handler](p.GetOCIDNSHandler()) + } + + if p, ok := cfg.(interface{ GetOCIDatabaseHandler() service.Registerable }); ok { + conf.OCIDatabaseOps, _ = getHandler[*ocidatabase.Handler](p.GetOCIDatabaseHandler()) + } + if p, ok := cfg.(interface{ GetOCIEventsHandler() service.Registerable }); ok { conf.OCIEventsOps, _ = getHandler[*ocievents.Handler](p.GetOCIEventsHandler()) } diff --git a/dashboard/static/icons/azure-entra.svg b/dashboard/static/icons/azure-entra.svg new file mode 100644 index 0000000000..284a7498ef --- /dev/null +++ b/dashboard/static/icons/azure-entra.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/azure-network.svg b/dashboard/static/icons/azure-network.svg new file mode 100644 index 0000000000..a916a74353 --- /dev/null +++ b/dashboard/static/icons/azure-network.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/oci-database.svg b/dashboard/static/icons/oci-database.svg new file mode 100644 index 0000000000..129550c538 --- /dev/null +++ b/dashboard/static/icons/oci-database.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/static/icons/oci-dns.svg b/dashboard/static/icons/oci-dns.svg new file mode 100644 index 0000000000..f1069aa133 --- /dev/null +++ b/dashboard/static/icons/oci-dns.svg @@ -0,0 +1,4 @@ + + + + diff --git a/dashboard/ui.go b/dashboard/ui.go index 21a764e3a4..03f7bbc88c 100644 --- a/dashboard/ui.go +++ b/dashboard/ui.go @@ -51,6 +51,7 @@ import ( azureeventhubsbackend "github.com/blackbirdworks/gopherstack/services/azureeventhubs" azurefilebackend "github.com/blackbirdworks/gopherstack/services/azurefile" azurefunctionsbackend "github.com/blackbirdworks/gopherstack/services/azurefunctions" + azuregraphbackend "github.com/blackbirdworks/gopherstack/services/azuregraph" azurekeyvaultbackend "github.com/blackbirdworks/gopherstack/services/azurekeyvault" azuremonitorbackend "github.com/blackbirdworks/gopherstack/services/azuremonitor" azurequeuebackend "github.com/blackbirdworks/gopherstack/services/azurequeue" @@ -153,6 +154,8 @@ import ( ocicontainerenginebackend "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" ocicontainerinstancesbackend "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocidatabasebackend "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ocidnsbackend "github.com/blackbirdworks/gopherstack/services/ocidns" ocieventsbackend "github.com/blackbirdworks/gopherstack/services/ocievents" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" @@ -554,6 +557,8 @@ type Config struct { AppSyncOps *appsyncbackend.Handler // AzureARMOps is dispatched in-process by the /dashboard/api/v1/azure/arm proxy. AzureARMOps *azurearmbackend.Handler + // AzureGraphOps is dispatched in-process by the /dashboard/api/v1/azure/graph proxy. + AzureGraphOps *azuregraphbackend.Handler // GCPStorageOps is dispatched in-process by the /dashboard/api/v1/gcp/storage proxy. GCPStorageOps *gcpstoragebackend.Handler // GCPBigQueryOps is dispatched in-process by the /dashboard/api/v1/gcp/bigquery proxy. @@ -630,6 +635,10 @@ type Config struct { OCIArtifactsOps *ociartifactsbackend.Handler // OCIContainerInstancesOps is dispatched in-process by its proxy. OCIContainerInstancesOps *ocicontainerinstancesbackend.Handler + // OCIDNSOps is dispatched in-process by its proxy. + OCIDNSOps *ocidnsbackend.Handler + // OCIDatabaseOps is dispatched in-process by its proxy. + OCIDatabaseOps *ocidatabasebackend.Handler // AzureFileOps is dispatched in-process by the /dashboard/api/v1/azure/file proxy. AzureFileOps *azurefilebackend.Handler // AzureFunctionsOps is called in-process by the /dashboard/api/v1/azure/functions proxy. diff --git a/desktop/go.mod b/desktop/go.mod index 7d2eccae26..6f75341b5e 100644 --- a/desktop/go.mod +++ b/desktop/go.mod @@ -44,13 +44,13 @@ require ( github.com/alecthomas/kong v1.16.1 // indirect github.com/alicebob/miniredis/v2 v2.40.0 // indirect github.com/andybalholm/brotli v1.2.6 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect github.com/aws/aws-sdk-go-v2/config v1.33.9 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/acm v1.50.4 // indirect github.com/aws/aws-sdk-go-v2/service/amplify v1.48.4 // indirect @@ -106,7 +106,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect github.com/aws/aws-sdk-go-v2/service/transcribe v1.66.4 // indirect github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 // indirect - github.com/aws/smithy-go v1.28.5 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/cedar-policy/cedar-go v1.8.0 // indirect @@ -116,6 +116,7 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/distribution/distribution/v3 v3.1.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2/v2 v2.8.4 // indirect @@ -127,6 +128,7 @@ require ( github.com/dop251/goja v0.0.0-20261012212121-0c308c5ed11b // indirect github.com/dustin/go-humanize v1.1.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-git/go-billy/v5 v5.9.2 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-mysql-org/go-mysql v1.16.0 // indirect @@ -189,6 +191,7 @@ require ( github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/rabbitmq/amqp091-go v1.15.0 // indirect + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect github.com/recolabs/gnata v0.6.1 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 // indirect @@ -198,6 +201,7 @@ require ( github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/shopspring/decimal v1.5.0 // indirect + github.com/sijms/go-ora/v2 v2.9.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect github.com/tidwall/gjson v1.20.0 // indirect github.com/tidwall/match v1.2.0 // indirect @@ -206,6 +210,8 @@ require ( github.com/twmb/franz-go/pkg/kmsg v1.15.0 // indirect github.com/twpayne/go-geom v1.7.0 // indirect github.com/vektah/gqlparser/v2 v2.5.62 // indirect + github.com/willscott/go-nfs v0.0.4 // indirect + github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/desktop/go.sum b/desktop/go.sum index be0874c61d..075737b77a 100644 --- a/desktop/go.sum +++ b/desktop/go.sum @@ -206,20 +206,22 @@ github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 h1:udoNKmbYngh4RpmURFjIaduwPjS/pTHkySWCTkS9Qf0= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8/go.mod h1:kAkwdYfRHYr653KcQh9vLui/BP4CFLrYvdD9npWb5Dc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 h1:m7EQZA6WxCbiExtCYq1wJLvK/dTQgQc1KvEOBnF9GlY= @@ -444,6 +446,8 @@ github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 h1:E9E99pv5u2Uc0VkgXp github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3/go.mod h1:IlKDS05ABuNxYQ/UspDTEoKVOmkpXne18fVJFMTOKeQ= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 h1:QcDoECg2br9JOaQ6C0fR3Mkcj9b/+dFSRuxZCLS3tME= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2/go.mod h1:GZ1UAuKbyxBdaRNbhKbAobG2VcWrEg5YmgWstJA3sl8= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 h1:0y77BcwEEKEuwUSDuJsxpUC0UYxK5iTjkwDVAFkxGe0= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5/go.mod h1:yLxwAiadnny6r4QkMjWn0oChHryB6xbSIpMdGkr7doA= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 h1:M0KgyGQkqxe/MHR5Lj1PZGgwnJOnfN5bCDh470sIJDg= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4/go.mod h1:cYtZI/MoJuzdSBYXg4enE8RkNzKRkoMr0qATq+ThsIw= github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 h1:hbejt97Z2rKhxcjlpLu85REd1/w9Z8omGQkXl68FEI4= @@ -632,8 +636,8 @@ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 h1:s3r96tGUdWFhIolHNwR9+ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4/go.mod h1:rupBg6/2hKMz1uCa7sd/RSMmLOXJQWgMzYIOB+Qi2EA= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 h1:GDr+18TEmx5ivPIaYqCWjVtPGSbcVZQkgX3PzhfzkKE= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3/go.mod h1:My/sZif0SANOKoH+IiyDW9AlWR2ITZQny7u7pTBYBMQ= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -666,6 +670,8 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -702,8 +708,10 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= +github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= +github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= +github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e h1:Lf/gRkoycfOBPa42vU2bbgPurFong6zXeFtPoxholzU= github.com/go-json-experiment/json v0.0.0-20251027170946-4849db3c2f7e/go.mod h1:uNVvRXArCGbZ508SxYYTC5v1JWoz2voff5pm25jU1Ok= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -853,6 +861,8 @@ github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/nicksnyder/go-i18n/v2 v2.5.0 h1:3wH1gpaekcgGuwzWdSu7JwJhH9Tk87k1ezt0i1p2/Is= github.com/nicksnyder/go-i18n/v2 v2.5.0/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -894,6 +904,8 @@ github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0o github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw= github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/recolabs/gnata v0.6.1 h1:2YCxrmUK64UFm1fPBFYe5jFHS6Q8EKJC+34QCujeU44= github.com/recolabs/gnata v0.6.1/go.mod h1:+u5DmoYMyHl+9Uuo9KQXLhbKHip+M5N8S/QYAPSG0XU= github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0= @@ -916,6 +928,8 @@ github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63 github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg= github.com/shopspring/decimal v1.5.0 h1:HxSWk31RmJv0EyY4oLiGQB9u7WPludUwPBdvQZnYFPc= github.com/shopspring/decimal v1.5.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sijms/go-ora/v2 v2.9.0 h1:+iQbUeTeCOFMb5BsOMgUhV8KWyrv9yjKpcK4x7+MFrg= +github.com/sijms/go-ora/v2 v2.9.0/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= @@ -956,6 +970,10 @@ github.com/vektah/gqlparser/v2 v2.5.62 h1:GJP/emj1wIWKs0U3SHcAgzknWgylTqQ1qH2gNL github.com/vektah/gqlparser/v2 v2.5.62/go.mod h1:JNK+plRwKdXLsF/qPFPe5tE0z4s1WeroD9S5LR8um/Q= github.com/wailsapp/wails/v3 v3.0.0 h1:lBe89kFu23bVz1dc1H9v0srkHQgWsy7BAKpH4sLQuDw= github.com/wailsapp/wails/v3 v3.0.0/go.mod h1:U3p3/QfSer2XxXM3XWhwis9O7LvvuNKN/AcGoEFD+K8= +github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI= +github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 h1:U0DnHRZFzoIV1oFEZczg5XyPut9yxk9jjtax/9Bxr/o= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= diff --git a/docker-compose.yml b/docker-compose.yml index 139bd57ca6..549602dbb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,10 @@ services: start_period: 5s volumes: - gopherstack-data:/data + # Docker-backed engines (Lambda, ECS, ...): the image runs as UID 65532, so also mount the socket and + # join its group, e.g. `DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up`. + # - /var/run/docker.sock:/var/run/docker.sock + # group_add: ["${DOCKER_GID}"] volumes: gopherstack-data: diff --git a/docs/docker.md b/docs/docker.md index aa7b9818d0..e549775205 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -80,6 +80,12 @@ environment: - ELASTICACHE_ENGINE=embedded ``` +## Running as non-root + +The image is built on `gcr.io/distroless/static:nonroot` and runs as `65532:65532` by default. The binary is `/usr/local/bin/gopherstack` (the working directory, so `./gopherstack health` keeps working), `/data` is owned by 65532 and `/tmp` is `1777`. The image used to be `FROM scratch` and ran as root with the binary at `/root/gopherstack`; that path is gone. + +To run as root again (for example to keep an older root-owned volume without a `chown`), pass `--user 0:0` / `user: "0:0"`. + ## Volume-backed persistence State snapshots are written to `/data` inside the container. Mount a named volume or host path to keep state across container restarts: @@ -90,6 +96,18 @@ volumes: - ./local-data:/data # host path ``` +A new named volume inherits the ownership of `/data` (65532) and just works. A host path keeps the host's ownership, so give the container user write access, using either of: + +```bash +mkdir -p local-data && sudo chown 65532:65532 local-data +docker run --rm -p 8000:8000 -e PERSIST=true -v "$PWD/local-data:/data" ghcr.io/blackbirdworks/gopherstack:latest + +# or run as your own user +docker run --rm --user "$(id -u):$(id -g)" -p 8000:8000 -e PERSIST=true -v "$PWD/local-data:/data" ghcr.io/blackbirdworks/gopherstack:latest +``` + +Without write access Gopherstack logs `mkdir /data/...: permission denied`. A named volume created by an older, root-running image is still owned by root; fix it once with `docker run --rm -v gopherstack-data:/data alpine chown -R 65532:65532 /data`. + ## Multi-service application example Combine Gopherstack with your application in a single Compose file: @@ -165,10 +183,19 @@ docker run --rm -p 8000:8000 gopherstack:local Lambda image-based functions require the Docker daemon. When running Gopherstack itself inside Docker, you need to mount the host Docker socket and set `LAMBDA_DOCKER_HOST` to the host IP: +The container user (65532) is not in the socket's group, so add that group with `--group-add`/`group_add`: + +```bash +docker run --rm -p 8000:8000 -v /var/run/docker.sock:/var/run/docker.sock \ + --group-add "$(stat -c %g /var/run/docker.sock)" ghcr.io/blackbirdworks/gopherstack:latest +``` + ```yaml services: gopherstack: image: ghcr.io/blackbirdworks/gopherstack:latest + group_add: + - "${DOCKER_GID}" # export DOCKER_GID=$(stat -c %g /var/run/docker.sock) environment: - LAMBDA_DOCKER_HOST=host-gateway # or your host IP volumes: @@ -178,6 +205,8 @@ services: - "8000:8000" ``` +On Docker Desktop the socket is typically `root:root` inside the VM; use `--group-add 0` or `--user 0:0`. Without the group, engine-backed services (Lambda, ECS, ...) cannot reach the daemon and stay metadata-only. + See [Lambda architecture](architecture/lambda.md) for details. ## Registry credentials and mirrors diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 0000000000..6c12174cf5 --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,82 @@ +# Kubernetes (Helm chart) + +`charts/gopherstack` deploys Gopherstack into a cluster. It renders a Deployment (one replica, `Recreate`), a Service, a ConfigMap of environment settings, and optionally a PVC, an Ingress and a Docker-in-Docker sidecar. + +## Install + +From a checkout: + +```bash +helm install gopherstack ./charts/gopherstack +kubectl port-forward svc/gopherstack-gopherstack 8000:8000 +AWS_ENDPOINT_URL=http://localhost:8000 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test \ + aws s3 mb s3://demo +``` + +The image is `ghcr.io/blackbirdworks/gopherstack`, tagged with the chart `appVersion` unless `image.tag` is set. In-cluster clients use `http://-gopherstack..svc.cluster.local:8000`. + +## Common values + +| Value | Default | Effect | +|---|---|---| +| `config.region`, `config.accountId`, `config.logLevel` | `us-east-1`, `000000000000`, `info` | `REGION`, `ACCOUNT_ID`, `LOG_LEVEL` | +| `config.services` / `config.disableServices` | `[]` | `GOPHERSTACK_SERVICES` / `GOPHERSTACK_DISABLE_SERVICES` | +| `config.engines` | `stub` | `GOPHERSTACK_ENGINES` (`stub`, `auto`, `docker`) | +| `config.enforceIam` | `false` | `GOPHERSTACK_ENFORCE_IAM` | +| `config.extra` | `{}` | any other env var, `KEY: value` | +| `persistence.enabled` | `false` | PVC mounted at `/data`, sets `PERSIST=true` and `GOPHERSTACK_DATA_DIR` | +| `service.type` / `service.port` | `ClusterIP` / `8000` | main endpoint (`PORT` follows `service.port`) | +| `service.extraPorts` | `[]` | extra named container and Service ports (below) | +| `ingress.*` | disabled | standard Ingress for the main port | +| `probes.*` | `/_gopherstack/health` | liveness, readiness and startup probes | +| `dind.enabled` | `false` | privileged Docker-in-Docker sidecar (below) | +| `resources`, `nodeSelector`, `tolerations`, `affinity` | empty | scheduling | + +`values.schema.json` validates the values at install time. + +## Persistence + +With `persistence.enabled=true` Gopherstack snapshots to the PVC on shutdown and restores on start, so state survives pod restarts. Without it, state is in memory. Use `persistence.existingClaim` to bring your own PVC. + +## Per-cloud ports + +AWS services share the main port. Azure, GCP and OCI emulators listen on dedicated ports, listed in `fixedServicePorts` in `internal/app/cli.go`. Expose the ones you need: + +```yaml +service: + extraPorts: + - name: azure-blob # AZURE_BLOB_PORT, default 10000 + port: 10000 + - name: cosmos # COSMOSDB_PORT, default 8081 + port: 8081 +``` + +Names must be at most 15 characters. A few defaults: Azure Blob 10000, Queue 10001, Table 10002, Service Bus 10003, Cosmos 8081, GCP Pub/Sub 8085, GCP Storage 4443, IoT MQTT 1883. If you change a port with `config.extra` (for example `AZURE_BLOB_PORT`), change the matching `extraPorts` entry. + +## Docker-backed engines + +Engines that start real containers (RDS, EKS, Lambda containers, ECS tasks, EC2 docker provider, ...) need a Docker daemon, which a pod does not have. By default the chart runs with `config.engines=stub`, so those services answer from metadata only. Mounting the node's Docker socket is not supported by the chart. + +To use them anyway, enable the sidecar: + +```yaml +dind: + enabled: true +config: + engines: docker +``` + +This adds a `docker:dind` container that runs **privileged** and exposes `tcp://127.0.0.1:2375` (no TLS, pod-local) as `DOCKER_HOST`. It needs a cluster that permits privileged pods and is not covered by the install test. A native Kubernetes pod executor for Lambda/ECS/EC2 is not implemented. + +## Security + +The container drops all capabilities, forbids privilege escalation, uses the `RuntimeDefault` seccomp profile and a read-only root filesystem (`/tmp` and `/data` are writable volumes). It runs as the image's non-root user (UID/GID 65532, `runAsNonRoot: true`); `fsGroup` 65532 makes the PVC writable. The release image is distroless `static:nonroot`, so its root directory is world-searchable under containerd as well. To run docker-backed engines from the cluster, enable the `dind` sidecar rather than mounting a host socket. It mounts no service account token and the chart creates no RBAC, because Gopherstack does not call the Kubernetes API. + +## Tests + +```bash +task helm-test # lint + golden templates (docker, pinned alpine/helm image) +task helm-install-test # installs into a real k3s cluster started with pkgs/k3sengine +``` + +Both run `go test ./test/helm/` with `GOPHERSTACK_TEST_DOCKER=1`. The install test loads a locally built image (`bin/gopherstack-linux` via `Dockerfile.test`, or `GOPHERSTACK_HELM_IMAGE`), installs the chart with persistence on a NodePort, checks `/_gopherstack/health`, creates and lists an S3 bucket, restarts the pod to prove persistence, then uninstalls. Refresh goldens with `GOPHERSTACK_UPDATE_GOLDEN=1`. diff --git a/examples/apigw-websocket-chat/docker-compose.yml b/examples/apigw-websocket-chat/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/apigw-websocket-chat/docker-compose.yml +++ b/examples/apigw-websocket-chat/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/cognito-api-auth/docker-compose.yml b/examples/cognito-api-auth/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/cognito-api-auth/docker-compose.yml +++ b/examples/cognito-api-auth/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/ddb-lambda-chain/docker-compose.yml b/examples/ddb-lambda-chain/docker-compose.yml index 457d22c557..ff0134c487 100644 --- a/examples/ddb-lambda-chain/docker-compose.yml +++ b/examples/ddb-lambda-chain/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/ec2-docker/docker-compose.yml b/examples/ec2-docker/docker-compose.yml index a995c31863..117e34dfb0 100644 --- a/examples/ec2-docker/docker-compose.yml +++ b/examples/ec2-docker/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" environment: # Run the EC2 service with the docker-backed compute provider so that # RunInstances actually spawns a container (sshd, ec2-user) instead of diff --git a/examples/kinesis-lambda-aggregator/docker-compose.yml b/examples/kinesis-lambda-aggregator/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/kinesis-lambda-aggregator/docker-compose.yml +++ b/examples/kinesis-lambda-aggregator/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/s3-lambda-processor/docker-compose.yml b/examples/s3-lambda-processor/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/s3-lambda-processor/docker-compose.yml +++ b/examples/s3-lambda-processor/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/examples/stepfunctions-order-workflow/docker-compose.yml b/examples/stepfunctions-order-workflow/docker-compose.yml index 57e31609fb..88539f36b9 100644 --- a/examples/stepfunctions-order-workflow/docker-compose.yml +++ b/examples/stepfunctions-order-workflow/docker-compose.yml @@ -1,6 +1,8 @@ services: gopherstack: build: ../.. + # The image runs as UID 65532; root can open the Docker socket without a group-add. + user: "0:0" ports: - "8000:8000" - "10000-10100:10000-10100" diff --git a/go.mod b/go.mod index 1f17120ca1..d335f93126 100644 --- a/go.mod +++ b/go.mod @@ -77,10 +77,11 @@ require ( github.com/apache/tinkerpop/gremlin-go/v3 v3.8.2 github.com/aws/aws-dax-go v1.2.15 github.com/aws/aws-sdk-go v1.55.8 - github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 github.com/aws/aws-sdk-go-v2/config v1.33.9 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 + github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 github.com/aws/aws-sdk-go-v2/service/account v1.42.2 @@ -188,6 +189,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/kms v1.61.4 github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 + github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 github.com/aws/aws-sdk-go-v2/service/managedblockchain v1.40.3 @@ -282,13 +284,14 @@ require ( github.com/aws/aws-sdk-go-v2/service/workmail v1.45.3 github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 - github.com/aws/smithy-go v1.28.5 + github.com/aws/smithy-go v1.28.7 github.com/blevesearch/snowballstem v0.9.0 github.com/cedar-policy/cedar-go v1.8.0 github.com/containerd/errdefs v1.0.0 github.com/distribution/distribution/v3 v3.1.2 github.com/dop251/goja v0.0.0-20261012212121-0c308c5ed11b github.com/eclipse/paho.mqtt.golang v1.5.1 + github.com/go-git/go-billy/v5 v5.9.2 github.com/go-mysql-org/go-mysql v1.16.0 github.com/go-sql-driver/mysql v1.10.1 github.com/go-stomp/stomp/v3 v3.1.5 @@ -317,11 +320,14 @@ require ( github.com/rabbitmq/amqp091-go v1.15.0 github.com/recolabs/gnata v0.6.1 github.com/redis/go-redis/v9 v9.23.0 + github.com/sijms/go-ora/v2 v2.9.0 github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.44.0 github.com/twmb/franz-go v1.22.1 github.com/twmb/franz-go/pkg/kmsg v1.15.0 github.com/vektah/gqlparser/v2 v2.5.62 + github.com/willscott/go-nfs v0.0.4 + github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 go.mongodb.org/mongo-driver/v2 v2.9.2 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.58.0 @@ -363,8 +369,8 @@ require ( github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e // indirect github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect @@ -382,6 +388,7 @@ require ( github.com/containerd/platforms v0.2.1 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2/v2 v2.8.4 // indirect @@ -458,6 +465,7 @@ require ( github.com/prometheus/common v0.72.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect + github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.23.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/go.sum b/go.sum index 2e68fcd70b..3887eca7fa 100644 --- a/go.sum +++ b/go.sum @@ -230,20 +230,22 @@ github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ= github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8 h1:udoNKmbYngh4RpmURFjIaduwPjS/pTHkySWCTkS9Qf0= +github.com/aws/aws-sdk-go-v2/feature/dsql/auth v1.2.8/go.mod h1:kAkwdYfRHYr653KcQh9vLui/BP4CFLrYvdD9npWb5Dc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.57.3 h1:m7EQZA6WxCbiExtCYq1wJLvK/dTQgQc1KvEOBnF9GlY= @@ -468,6 +470,8 @@ github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3 h1:E9E99pv5u2Uc0VkgXp github.com/aws/aws-sdk-go-v2/service/lakeformation v1.55.3/go.mod h1:IlKDS05ABuNxYQ/UspDTEoKVOmkpXne18fVJFMTOKeQ= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2 h1:QcDoECg2br9JOaQ6C0fR3Mkcj9b/+dFSRuxZCLS3tME= github.com/aws/aws-sdk-go-v2/service/lambda v1.112.2/go.mod h1:GZ1UAuKbyxBdaRNbhKbAobG2VcWrEg5YmgWstJA3sl8= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5 h1:0y77BcwEEKEuwUSDuJsxpUC0UYxK5iTjkwDVAFkxGe0= +github.com/aws/aws-sdk-go-v2/service/lambdaweb v1.2.5/go.mod h1:yLxwAiadnny6r4QkMjWn0oChHryB6xbSIpMdGkr7doA= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4 h1:M0KgyGQkqxe/MHR5Lj1PZGgwnJOnfN5bCDh470sIJDg= github.com/aws/aws-sdk-go-v2/service/lightsail v1.66.4/go.mod h1:cYtZI/MoJuzdSBYXg4enE8RkNzKRkoMr0qATq+ThsIw= github.com/aws/aws-sdk-go-v2/service/macie2 v1.59.3 h1:hbejt97Z2rKhxcjlpLu85REd1/w9Z8omGQkXl68FEI4= @@ -656,8 +660,8 @@ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4 h1:s3r96tGUdWFhIolHNwR9+ github.com/aws/aws-sdk-go-v2/service/workspaces v1.81.4/go.mod h1:rupBg6/2hKMz1uCa7sd/RSMmLOXJQWgMzYIOB+Qi2EA= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3 h1:GDr+18TEmx5ivPIaYqCWjVtPGSbcVZQkgX3PzhfzkKE= github.com/aws/aws-sdk-go-v2/service/xray v1.45.3/go.mod h1:My/sZif0SANOKoH+IiyDW9AlWR2ITZQny7u7pTBYBMQ= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= @@ -697,6 +701,8 @@ github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GK github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -751,6 +757,8 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-git/go-billy/v5 v5.9.2 h1:OXFSRyz4g20upsGDJgQG9Bak1l/ZEv8GHVYB52O71sE= +github.com/go-git/go-billy/v5 v5.9.2/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4= github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -943,6 +951,8 @@ github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= github.com/nicksnyder/go-i18n/v2 v2.5.0 h1:3wH1gpaekcgGuwzWdSu7JwJhH9Tk87k1ezt0i1p2/Is= github.com/nicksnyder/go-i18n/v2 v2.5.0/go.mod h1:DrhgsSDZxoAfvVrBVLXoxZn/pN5TXqaDbq7ju94viiQ= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -989,6 +999,8 @@ github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0o github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rabbitmq/amqp091-go v1.15.0 h1:LEQL4/yp48/Wigt6A6XOu18RQRo8ZHtB5I/KZJn+gkw= github.com/rabbitmq/amqp091-go v1.15.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= +github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= github.com/recolabs/gnata v0.6.1 h1:2YCxrmUK64UFm1fPBFYe5jFHS6Q8EKJC+34QCujeU44= github.com/recolabs/gnata v0.6.1/go.mod h1:+u5DmoYMyHl+9Uuo9KQXLhbKHip+M5N8S/QYAPSG0XU= github.com/redis/go-redis/extra/rediscmd/v9 v9.23.0 h1:YOQCtePRR/cBrxtZB5u2ueYei3zGJm1n0KB2tvth0k0= @@ -1011,6 +1023,8 @@ github.com/shirou/gopsutil/v4 v4.26.9 h1:CaBo/hBFqvlJoLvAQEKdHdf5GD97/MN2ACfyv63 github.com/shirou/gopsutil/v4 v4.26.9/go.mod h1:nKH+8wX2zxr/mDbsR+AAy6Qd2z4ZOQ7wfQaEp+yrYmg= github.com/shopspring/decimal v1.5.0 h1:HxSWk31RmJv0EyY4oLiGQB9u7WPludUwPBdvQZnYFPc= github.com/shopspring/decimal v1.5.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sijms/go-ora/v2 v2.9.0 h1:+iQbUeTeCOFMb5BsOMgUhV8KWyrv9yjKpcK4x7+MFrg= +github.com/sijms/go-ora/v2 v2.9.0/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= @@ -1052,6 +1066,10 @@ github.com/twpayne/go-geom v1.7.0 h1:zMHTHNlenoHAc2ZSiHFh7SMoG0Wdw8gF0orBnrBhfjE github.com/twpayne/go-geom v1.7.0/go.mod h1:y9Q2YcKKaC2PHXwtRvwEunOPx5Oa+U8aAxPmFvscROk= github.com/vektah/gqlparser/v2 v2.5.62 h1:GJP/emj1wIWKs0U3SHcAgzknWgylTqQ1qH2gNL+wSWc= github.com/vektah/gqlparser/v2 v2.5.62/go.mod h1:JNK+plRwKdXLsF/qPFPe5tE0z4s1WeroD9S5LR8um/Q= +github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI= +github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00 h1:U0DnHRZFzoIV1oFEZczg5XyPut9yxk9jjtax/9Bxr/o= +github.com/willscott/go-nfs-client v0.0.0-20240104095149-b44639837b00/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= diff --git a/internal/app/cli.go b/internal/app/cli.go index acc58566b0..e9a1ef723e 100644 --- a/internal/app/cli.go +++ b/internal/app/cli.go @@ -222,6 +222,7 @@ import ( kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" lakeformationbackend "github.com/blackbirdworks/gopherstack/services/lakeformation" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" lightsailbackend "github.com/blackbirdworks/gopherstack/services/lightsail" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" managedblockchainbackend "github.com/blackbirdworks/gopherstack/services/managedblockchain" @@ -247,7 +248,10 @@ import ( ocioke "github.com/blackbirdworks/gopherstack/services/ocicontainerengine" ocicontainerinstances "github.com/blackbirdworks/gopherstack/services/ocicontainerinstances" ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + ocidatabase "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ocidns "github.com/blackbirdworks/gopherstack/services/ocidns" ocievents "github.com/blackbirdworks/gopherstack/services/ocievents" + ocifilestorage "github.com/blackbirdworks/gopherstack/services/ocifilestorage" ocifunctionsbackend "github.com/blackbirdworks/gopherstack/services/ocifunctions" ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" ociloadbalancer "github.com/blackbirdworks/gopherstack/services/ociloadbalancer" @@ -451,6 +455,7 @@ type CLI struct { cloudtrailHandler service.Registerable azureARMHandler service.Registerable azureBlobHandler service.Registerable + azureGraphHandler service.Registerable gcpStorageHandler service.Registerable gcpPubSubHandler service.Registerable gcpFirestoreHandler service.Registerable @@ -499,6 +504,9 @@ type CLI struct { ociLoadBalancerHandler service.Registerable ociArtifactsHandler service.Registerable ociContainerInstancesHandler service.Registerable + ociDNSHandler service.Registerable + ociDatabaseHandler service.Registerable + ociFileStorageHandler service.Registerable azureMonitorHandler service.Registerable ociVaultHandler service.Registerable ociStreamingHandler service.Registerable @@ -651,6 +659,9 @@ type CLI struct { AzureGraph azuregraphbackend.Settings `embed:"" prefix:"azure-graph-"` CodeBuild codebuildbackend.Settings `embed:"" prefix:"codebuild-"` Glue gluebackend.Settings `embed:"" prefix:"glue-"` + DSQL dsqlbackend.Settings `embed:"" prefix:"dsql-"` + LambdaWeb lambdawebbackend.Settings `embed:"" prefix:"lambdaweb-"` + Transcribe transcribebackend.Settings `embed:"" prefix:"transcribe-"` GCPIdentityToolkit gcpidtkbackend.Settings `embed:"" prefix:"gcp-identitytoolkit-"` AzureKeyVault azurekeyvaultbackend.Settings `embed:"" prefix:"azure-keyvault-"` S3 s3backend.Settings `embed:"" prefix:"s3-"` @@ -665,6 +676,8 @@ type CLI struct { OCIContainerEngine ocioke.Settings `embed:"" prefix:"oci-oke-"` OCIContainerInstances ocicontainerinstances.Settings `embed:""` OCIMySQL ocimysqlbackend.Settings `embed:"" prefix:"oci-mysql-"` + OCIDatabase ocidatabase.Settings `embed:"" prefix:"oci-database-"` + OCIFileStorage ocifilestorage.Settings `embed:"" prefix:"oci-filestorage-"` AgentCore agentcorebackend.Settings `embed:"" prefix:"agentcore-"` GCPCloudBuild gcpcloudbuildbackend.Settings `embed:"" prefix:"gcp-cloudbuild-"` GCPBatch gcpbatchbackend.Settings `embed:"" prefix:"gcp-batch-"` @@ -754,6 +767,7 @@ type CLI struct { OCIEvents ocievents.Settings `embed:"" prefix:"oci-events-"` OCILoadBalancer ociloadbalancer.Settings `embed:"" prefix:"oci-loadbalancer-"` OCIArtifacts ociartifacts.Settings `embed:"" prefix:"oci-artifacts-"` + OCIDNS ocidns.Settings `embed:"" prefix:"oci-dns-"` } // GetGlobalConfig returns the centralised account ID and region (config.Provider). @@ -999,6 +1013,11 @@ func (c *CLI) GetOCIMySQLEngine() string { return c.engineMode("oci-mysql-engine", c.OCIMySQL.Engine) } +// GetOCIDatabaseEngine returns the OCI Database engine mode (ocidatabase.EngineConfigProvider). +func (c *CLI) GetOCIDatabaseEngine() string { + return c.engineMode("oci-database-engine", c.OCIDatabase.Engine) +} + // GetOCINotificationsSettings returns OCI Notifications settings (ocinotifications.ConfigProvider). func (c *CLI) GetOCINotificationsSettings() ocinotificationsbackend.Settings { return c.OCINotifications @@ -1505,6 +1524,30 @@ func (c *CLI) GetGlueSettings() gluebackend.Settings { return st } +// GetDSQLSettings returns DSQL settings (dsql.ConfigProvider). +func (c *CLI) GetDSQLSettings() dsqlbackend.Settings { + st := c.DSQL + st.Engine = c.engineMode("dsql-engine", st.Engine) + + return st +} + +// GetLambdaWebSettings returns Lambda Web Functions settings (lambdaweb.ConfigProvider). +func (c *CLI) GetLambdaWebSettings() lambdawebbackend.Settings { + st := c.LambdaWeb + st.Engine = c.engineMode("lambdaweb-engine", st.Engine) + + return st +} + +// GetTranscribeSettings returns Transcribe settings (transcribe.ConfigProvider). +func (c *CLI) GetTranscribeSettings() transcribebackend.Settings { + st := c.Transcribe + st.Engine = c.engineMode("transcribe-engine", st.Engine) + + return st +} + // GetEC2Settings returns EC2 settings (ec2.ConfigProvider). func (c *CLI) GetEC2Settings() ec2backend.Settings { return c.EC2 @@ -2793,6 +2836,10 @@ func fixedServicePorts(cli CLI) []fixedServicePort { {"OCILoadBalancer", "ociloadbalancer", cli.OCILoadBalancer.Port}, {"OCIArtifacts", "ociartifacts", cli.OCIArtifacts.Port}, {"OCIContainerInstances", "ocicontainerinstances", cli.OCIContainerInstances.Port}, + {"OCIDNS", "ocidns", cli.OCIDNS.Port}, + {"OCIDNSServer", "ocidns-server", cli.OCIDNS.DNSPort}, + {"OCIDatabase", "ocidatabase", cli.OCIDatabase.Port}, + {"OCIFileStorage", "ocifilestorage", cli.OCIFileStorage.Port}, } } @@ -3166,6 +3213,7 @@ func buildEchoServer( registerSSOEndpoints(e, services) registerEC2NetworkEndpoints(e, services) registerEKSEndpoints(e, services) + registerTranscribeEndpoints(e, services) registerPricingEndpoints(e, services) e.POST("/_gopherstack/snapshot", buildSnapshotHandler(persistManager)) e.POST("/_gopherstack/load", buildLoadHandler(persistManager)) @@ -3528,6 +3576,9 @@ func storeCLIRecentHandlers(cli *CLI, byName map[string]service.Registerable) { cli.ociLoadBalancerHandler = byName["OCILoadBalancer"] cli.ociArtifactsHandler = byName["OCIArtifacts"] cli.ociContainerInstancesHandler = byName["OCIContainerInstances"] + cli.ociDNSHandler = byName["OCIDNS"] + cli.ociDatabaseHandler = byName["OCIDatabase"] + cli.ociFileStorageHandler = byName["OCIFileStorage"] storeAdditionalCLIHandlers(cli, byName) } @@ -3810,6 +3861,9 @@ func wireCrossServiceDependencies( wireOCILoadBalancer(byName) wireOCIArtifacts(byName) wireOCIContainerInstances(byName) + wireOCIDNS(byName) + wireOCIDatabase(byName) + wireOCIFileStorage(byName) } // wireOCIStreamingVault lets OCI Streaming validate customEncryptionKey references against OCI Vault keys. @@ -5269,6 +5323,7 @@ func getLatestServiceProviders() []service.Provider { // Extracted from getServiceProviders to satisfy the funlen limit. func getNewestServiceProviders() []service.Provider { return append([]service.Provider{ + &lambdawebbackend.Provider{}, &mwaabackend.Provider{}, &neptunebackend.Provider{}, &signinbackend.Provider{}, @@ -5314,6 +5369,9 @@ func getLeadingRecentServiceProviders() []service.Provider { &ociloadbalancer.Provider{}, &ociartifacts.Provider{}, &ocicontainerinstances.Provider{}, + &ocidns.Provider{}, + &ocidatabase.Provider{}, + &ocifilestorage.Provider{}, &gcppubsubbackend.Provider{}, &gcpsecretmanagerbackend.Provider{}, &gcpkmsbackend.Provider{}, @@ -13193,6 +13251,7 @@ func setupChaosAndRegistry( wireCloudControlHandlers(e, services) wireServiceRoleAuthorizer(services, cli.EnforceIAM) wireRDSIAMAuth(e, services, cli.EnforceIAM) + wireDSQLAuth(e, services, cli.EnforceIAM) wireNeptuneIAMAuth(services, cli.EnforceIAM) wireDocDBCA(e, services) wireKafkaAuth(e, services, cli.EnforceIAM) diff --git a/internal/app/cloud_dashboard_handlers.go b/internal/app/cloud_dashboard_handlers.go index 30be537570..5274349779 100644 --- a/internal/app/cloud_dashboard_handlers.go +++ b/internal/app/cloud_dashboard_handlers.go @@ -52,6 +52,7 @@ func storeConsoleHandlers(cli *CLI, byName map[string]service.Registerable) { cli.gcpCloudBuildHandler = byName["GCPCloudBuild"] cli.gcpRedisHandler = byName["GCPRedis"] cli.gcpMonitoringHandler = byName["GCPMonitoring"] + cli.azureGraphHandler = byName["AzureGraph"] } // GetGCPWorkflowsHandler returns the GCPWorkflows handler for the dashboard proxy. @@ -193,3 +194,8 @@ func (c *CLI) GetGCPResourceManagerHandler() service.Registerable { return c.gcp // //nolint:ireturn // architecturally required to return interface func (c *CLI) GetGCPBatchHandler() service.Registerable { return c.gcpBatchHandler } + +// GetAzureGraphHandler returns the Microsoft Graph handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetAzureGraphHandler() service.Registerable { return c.azureGraphHandler } diff --git a/internal/app/cloudformation_settings.go b/internal/app/cloudformation_settings.go index bad6d4f0b7..beb6567b35 100644 --- a/internal/app/cloudformation_settings.go +++ b/internal/app/cloudformation_settings.go @@ -1,9 +1,21 @@ package app -// CloudFormationSettings configures CloudFormation template validation. +import "time" + +// DefaultCloudFormationProvisioningDelay is how long each stack step stays *_IN_PROGRESS. +const DefaultCloudFormationProvisioningDelay = 300 * time.Millisecond + +// CloudFormationSettings configures CloudFormation template validation and provisioning. type CloudFormationSettings struct { + ProvisioningDelay time.Duration `name:"provisioning-delay" env:"GOPHERSTACK_CFN_PROVISIONING_DELAY" default:"300ms" help:"How long each stack step stays in *_IN_PROGRESS before completing, like real CloudFormation. 0 completes stacks instantly."` //nolint:lll // config struct tags are intentionally verbose + ValidateProperties bool `name:"validate-properties" env:"GOPHERSTACK_CLOUDFORMATION_VALIDATE_PROPERTIES" default:"true" negatable:"" help:"Reject unknown, wrong-typed and missing required resource properties at stack operations, like CloudFormation. Disable to accept any property."` //nolint:lll // config struct tags are intentionally verbose } // GetCloudFormationValidateProperties reports whether resource property validation is on. func (c *CLI) GetCloudFormationValidateProperties() bool { return c.CloudFormation.ValidateProperties } + +// GetCloudFormationProvisioningDelay returns the per-step *_IN_PROGRESS duration. +func (c *CLI) GetCloudFormationProvisioningDelay() time.Duration { + return c.CloudFormation.ProvisioningDelay +} diff --git a/internal/app/cloudformation_settings_test.go b/internal/app/cloudformation_settings_test.go index bfdcd43163..13b942e4c1 100644 --- a/internal/app/cloudformation_settings_test.go +++ b/internal/app/cloudformation_settings_test.go @@ -2,6 +2,7 @@ package app import ( "testing" + "time" "github.com/alecthomas/kong" "github.com/stretchr/testify/assert" @@ -37,3 +38,45 @@ func TestCloudFormationValidatePropertiesFlag(t *testing.T) { }) } } + +func TestCloudFormationProvisioningDelayFlag(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want time.Duration + }{ + {name: "default", want: DefaultCloudFormationProvisioningDelay}, + {name: "instant", args: []string{"--cloudformation-provisioning-delay=0"}}, + {name: "custom", args: []string{"--cloudformation-provisioning-delay=2s"}, want: 2 * time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var root rootCLI + + parser, err := kong.New(&root, kong.Name("gopherstack"), kong.Writers(nil, nil)) + require.NoError(t, err) + _, err = parser.Parse(tt.args) + require.NoError(t, err) + + assert.Equal(t, tt.want, root.Serve.GetCloudFormationProvisioningDelay()) + }) + } +} + +func TestCloudFormationProvisioningDelayEnv(t *testing.T) { + t.Setenv("GOPHERSTACK_CFN_PROVISIONING_DELAY", "750ms") + + var root rootCLI + + parser, err := kong.New(&root, kong.Name("gopherstack"), kong.Writers(nil, nil)) + require.NoError(t, err) + _, err = parser.Parse(nil) + require.NoError(t, err) + + assert.Equal(t, 750*time.Millisecond, root.Serve.GetCloudFormationProvisioningDelay()) +} diff --git a/internal/app/dms_rules_docker_test.go b/internal/app/dms_rules_docker_test.go new file mode 100644 index 0000000000..da6f1b627f --- /dev/null +++ b/internal/app/dms_rules_docker_test.go @@ -0,0 +1,324 @@ +package app + +import ( + "fmt" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice" + dmstypes "github.com/aws/aws-sdk-go-v2/service/databasemigrationservice/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const filterItemsRule = `{"rule-type":"selection","rule-id":"1","rule-name":"1","rule-action":"include",` + + `"object-locator":{"schema-name":"appdb","table-name":"items"},"filters":[` + + `{"filter-type":"source","column-name":"id","filter-conditions":[{"filter-operator":"ste","value":"5"}]},` + + `{"filter-type":"source","column-name":"qty","filter-conditions":[{"filter-operator":"gte","value":"10"}]}]}` + +func columnRule(id, action, column, extra string) string { + return fmt.Sprintf(`{"rule-type":"transformation","rule-id":"%s","rule-name":"%s","rule-target":"column",`+ + `"rule-action":"%s","object-locator":{"schema-name":"appdb","table-name":"items","column-name":"%s"}%s}`, + id, id, action, column, extra) +} + +func TestDMSDockerFiltersAndColumnRules(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec(t, src, + "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40), qty INT)", + "CREATE TABLE logs (id INT PRIMARY KEY)", + "INSERT INTO items VALUES (1,'alpha',10),(2,'beta',5),(3,'gamma',30),(6,'delta',60)", + "INSERT INTO logs VALUES (1)") + + ri := h.instance(t) + srcARN := h.endpoint(t, "src-ep", dmstypes.ReplicationEndpointTypeValueSource, "mysql", &srcEP, nil) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + mapping := mappingRules( + filterItemsRule, + `{"rule-type":"selection","rule-id":"2","rule-name":"2","rule-action":"include",`+ + `"object-locator":{"schema-name":"appdb","table-name":"log_"}}`, + columnRule("3", "add-column", "label", + `,"value":"label","expression":"upper($name)||'-'||$id","data-type":{"type":"string","length":30}`), + columnRule("4", "change-data-type", "qty", `,"data-type":{"type":"string","length":"12"}`), + ) + + taskARN := h.task(t, "rules", srcARN, tgtARN, ri, "full-load-and-cdc", mapping) + h.awaitStatus(t, taskARN, "running") + + const q = "SELECT id::text, name, qty, label FROM appdb.items ORDER BY id" + + awaitRows(t, tgt, q, [][]string{{"1", "alpha", "10", "ALPHA-1"}, {"3", "gamma", "30", "GAMMA-3"}}) + awaitRows(t, tgt, "SELECT id::text FROM appdb.logs", [][]string{{"1"}}) + + types := queryRows(t, tgt, "SELECT column_name, data_type, COALESCE(character_maximum_length::text,'') "+ + "FROM information_schema.columns WHERE table_schema='appdb' AND table_name='items' ORDER BY ordinal_position") + assert.Equal(t, [][]string{ + {"id", "integer", ""}, {"name", "character varying", "40"}, {"qty", "character varying", "12"}, + {"label", "character varying", "30"}, + }, types) + + mustExec(t, src, + "INSERT INTO items VALUES (4,'epsilon',40),(5,'low',1),(9,'high-id',90)", + "UPDATE items SET name='ALPHA2' WHERE id=1", + "UPDATE items SET qty=1 WHERE id=3", + "UPDATE items SET qty=70 WHERE id=2") + awaitRows(t, tgt, q, [][]string{ + {"1", "ALPHA2", "10", "ALPHA2-1"}, {"2", "beta", "70", "BETA-2"}, {"4", "epsilon", "40", "EPSILON-4"}, + }) + + h.stop(t, taskARN) +} + +func TestDMSDockerS3SourceCSVAndCDC(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "postgres:16") + + h := newDMSHarness(t) + tgtEP := h.createDB(t, "tgt", "postgres", "16") + tgt := openDB(t, "postgres", tgtEP) + + s3c := s3.NewFromConfig(h.cfg, func(o *s3.Options) { o.UsePathStyle = true }) + _, err := s3c.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String("dms-in")}) + require.NoError(t, err) + + put := func(key, body string) { + _, perr := s3c.PutObject(t.Context(), &s3.PutObjectInput{ + Bucket: aws.String("dms-in"), Key: aws.String(key), Body: strings.NewReader(body), + }) + require.NoError(t, perr) + } + + put("src/hr/employee/LOAD001.csv", "101,Smith,2014-06-04 10:00:00.123,12.50\n102,Jones,2015-10-08 00:00:00.000,7\n") + + const definition = `{"TableCount":"1","Tables":[{"TableName":"employee","TablePath":"hr/employee/",` + + `"TableOwner":"hr","TableColumns":[` + + `{"ColumnName":"Id","ColumnType":"INT8","ColumnNullable":"false","ColumnIsPk":"true"},` + + `{"ColumnName":"LastName","ColumnType":"STRING","ColumnLength":"20","ColumnNullable":"true"},` + + `{"ColumnName":"HireDate","ColumnType":"DATETIME","ColumnScale":"3","ColumnNullable":"true"},` + + `{"ColumnName":"Rate","ColumnType":"NUMERIC","ColumnPrecision":"5","ColumnScale":"2","ColumnNullable":"true"}],` + + `"TableColumnsTotal":"4"}]}` + + ri := h.instance(t) + srcARN := h.endpoint(t, "s3-src", dmstypes.ReplicationEndpointTypeValueSource, "s3", nil, + func(in *databasemigrationservice.CreateEndpointInput) { + in.ExternalTableDefinition = aws.String(definition) + in.S3Settings = &dmstypes.S3Settings{ + BucketName: aws.String("dms-in"), BucketFolder: aws.String("src"), CdcPath: aws.String("changes"), + ServiceAccessRoleArn: aws.String(dmsRoleARN), + } + }) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + conn, err := h.dms.TestConnection(t.Context(), &databasemigrationservice.TestConnectionInput{ + ReplicationInstanceArn: aws.String(ri), EndpointArn: aws.String(srcARN), + }) + require.NoError(t, err) + assert.Equal(t, "successful", aws.ToString(conn.Connection.Status)) + + taskARN := h.task(t, "s3-to-pg", srcARN, tgtARN, ri, "full-load-and-cdc", `{"rules":[`+ + `{"rule-type":"selection","rule-id":"1","rule-name":"1","rule-action":"include",`+ + `"object-locator":{"schema-name":"hr","table-name":"employee"}}]}`) + h.awaitStatus(t, taskARN, "running") + + const q = `SELECT "Id"::text, "LastName", to_char("HireDate",'YYYY-MM-DD HH24:MI:SS.MS'), "Rate"::text ` + + `FROM hr.employee ORDER BY "Id"` + + awaitRows(t, tgt, q, [][]string{ + {"101", "Smith", "2014-06-04 10:00:00.123", "12.50"}, {"102", "Jones", "2015-10-08 00:00:00.000", "7.00"}, + }) + + put("src/changes/CDC00001.csv", + "INSERT,employee,hr,103,Brown,2017-03-13 00:00:00.000,9.99\n"+ + "U,employee,hr,101,Smith-Updated,2014-06-04 10:00:00.123,13.00\n"+ + "D,employee,hr,102,Jones,2015-10-08 00:00:00.000,7.00\n") + awaitRows(t, tgt, q, [][]string{ + { + "101", + "Smith-Updated", + "2014-06-04 10:00:00.123", + "13.00", + }, {"103", "Brown", "2017-03-13 00:00:00.000", "9.99"}, + }) + + require.Eventually(t, func() bool { + s := h.tableStats(t, taskARN)["hr.employee"] + + return s.FullLoadRows == 2 && s.Inserts == 1 && s.Updates == 1 && s.Deletes == 1 + }, dmsDockerWait, dmsPoll) + + h.stop(t, taskARN) +} + +func TestDMSDockerEndpointFromSecretsManager(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec( + t, + src, + "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40))", + "INSERT INTO items VALUES (1,'alpha')", + ) + + sm := secretsmanager.NewFromConfig(h.cfg) + secret := func(name, password string) string { + out, err := sm.CreateSecret(t.Context(), &secretsmanager.CreateSecretInput{ + Name: aws.String(name), SecretString: aws.String(fmt.Sprintf( + `{"username":"master","password":%q,"host":%q,"port":"%d"}`, password, srcEP.host, srcEP.port)), + }) + require.NoError(t, err) + + return aws.ToString(out.ARN) + } + + secretEndpoint := func(id, secretID string) string { + return h.endpoint(t, id, dmstypes.ReplicationEndpointTypeValueSource, "mysql", nil, + func(in *databasemigrationservice.CreateEndpointInput) { + in.MySQLSettings = &dmstypes.MySQLSettings{ + DatabaseName: aws.String("appdb"), SecretsManagerSecretId: aws.String(secretID), + SecretsManagerAccessRoleArn: aws.String(dmsRoleARN), + } + }) + } + + ri := h.instance(t) + goodARN := secretEndpoint("good-ep", secret("dms/good", dmsMasterPass)) + badARN := secretEndpoint("bad-ep", secret("dms/bad", "wrong-password-xyz")) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + test := func(arn string) *dmstypes.Connection { + out, err := h.dms.TestConnection(t.Context(), &databasemigrationservice.TestConnectionInput{ + ReplicationInstanceArn: aws.String(ri), EndpointArn: aws.String(arn), + }) + require.NoError(t, err) + + return out.Connection + } + + assert.Equal(t, "successful", aws.ToString(test(goodARN).Status)) + + bad := test(badARN) + assert.Equal(t, "failed", aws.ToString(bad.Status)) + assert.NotContains(t, aws.ToString(bad.LastFailureMessage), "wrong-password-xyz") + assert.NotContains(t, aws.ToString(bad.LastFailureMessage), dmsMasterPass) + + taskARN := h.task(t, "secret-task", goodARN, tgtARN, ri, "full-load", tableMapping("items")) + h.awaitStatus(t, taskARN, "stopped") + assert.Equal(t, [][]string{{"1", "alpha"}}, queryRows(t, tgt, "SELECT id::text, name FROM appdb.items")) +} + +func TestDMSDockerServerlessReplication(t *testing.T) { + t.Parallel() + requireDMSDocker(t, "mysql:8.4", "postgres:16") + + h := newDMSHarness(t) + srcEP, tgtEP := h.createDB(t, "src", "mysql", "8.4"), h.createDB(t, "tgt", "postgres", "16") + src, tgt := openDB(t, "mysql", srcEP), openDB(t, "postgres", tgtEP) + + mustExec(t, src, "CREATE TABLE items (id INT PRIMARY KEY, name VARCHAR(40))", + "INSERT INTO items VALUES (1,'alpha'),(2,'beta')") + + srcARN := h.endpoint(t, "src-ep", dmstypes.ReplicationEndpointTypeValueSource, "mysql", &srcEP, nil) + tgtARN := h.endpoint(t, "tgt-ep", dmstypes.ReplicationEndpointTypeValueTarget, "postgres", &tgtEP, nil) + + cfg, err := h.dms.CreateReplicationConfig(t.Context(), &databasemigrationservice.CreateReplicationConfigInput{ + ReplicationConfigIdentifier: aws.String( + "serverless", + ), ReplicationType: dmstypes.MigrationTypeValueFullLoadAndCdc, + SourceEndpointArn: aws.String(srcARN), TargetEndpointArn: aws.String(tgtARN), + TableMappings: aws.String(tableMapping("items")), + ComputeConfig: &dmstypes.ComputeConfig{MinCapacityUnits: aws.Int32(2), MaxCapacityUnits: aws.Int32(8)}, + }) + require.NoError(t, err) + + cfgARN := aws.ToString(cfg.ReplicationConfig.ReplicationConfigArn) + replication := func() dmstypes.Replication { + out, derr := h.dms.DescribeReplications(t.Context(), &databasemigrationservice.DescribeReplicationsInput{ + Filters: []dmstypes.Filter{{Name: aws.String("replication-config-arn"), Values: []string{cfgARN}}}, + }) + require.NoError(t, derr) + require.Len(t, out.Replications, 1) + + return out.Replications[0] + } + await := func(status string) dmstypes.Replication { + var r dmstypes.Replication + + require.Eventually(t, func() bool { + r = replication() + if aws.ToString(r.Status) == "failed" { + require.FailNow(t, "replication failed: "+strings.Join(r.FailureMessages, "; ")) + } + + return aws.ToString(r.Status) == status + }, dmsDockerWait, dmsPoll) + + return r + } + start := func(kind string) { + _, serr := h.dms.StartReplication(t.Context(), &databasemigrationservice.StartReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), StartReplicationType: aws.String(kind), + }) + require.NoError(t, serr) + } + + start("start-replication") + await("running") + + const q = "SELECT id::text, name FROM appdb.items ORDER BY id" + + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "beta"}}) + + mustExec(t, src, "INSERT INTO items VALUES (3,'gamma')", "UPDATE items SET name='BETA' WHERE id=2") + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "BETA"}, {"3", "gamma"}}) + + require.Eventually(t, func() bool { + stats, serr := h.dms.DescribeReplicationTableStatistics(t.Context(), + &databasemigrationservice.DescribeReplicationTableStatisticsInput{ReplicationConfigArn: aws.String(cfgARN)}) + if serr != nil || len(stats.ReplicationTableStatistics) != 1 { + return false + } + + s := stats.ReplicationTableStatistics[0] + + return s.FullLoadRows == 2 && s.Inserts == 1 && s.Updates == 1 + }, dmsDockerWait, dmsPoll) + + running := replication() + require.NotNil(t, running.ReplicationStats) + assert.EqualValues(t, 1, running.ReplicationStats.TablesLoaded) + assert.EqualValues(t, 100, running.ReplicationStats.FullLoadProgressPercent) + require.NotNil(t, running.ProvisionData) + assert.EqualValues(t, 2, running.ProvisionData.ProvisionedCapacityUnits) + + _, err = h.dms.StopReplication(t.Context(), &databasemigrationservice.StopReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), + }) + require.NoError(t, err) + assert.Equal(t, "Stop Reason NORMAL", aws.ToString(await("stopped").StopReason)) + + mustExec(t, src, "INSERT INTO items VALUES (4,'delta')") + start("resume-processing") + await("running") + awaitRows(t, tgt, q, [][]string{{"1", "alpha"}, {"2", "BETA"}, {"3", "gamma"}, {"4", "delta"}}) + + _, err = h.dms.StopReplication(t.Context(), &databasemigrationservice.StopReplicationInput{ + ReplicationConfigArn: aws.String(cfgARN), + }) + require.NoError(t, err) + await("stopped") +} diff --git a/internal/app/dms_stores.go b/internal/app/dms_stores.go index 5a9ded327e..c62d77f559 100644 --- a/internal/app/dms_stores.go +++ b/internal/app/dms_stores.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -17,6 +18,7 @@ import ( ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" s3backend "github.com/blackbirdworks/gopherstack/services/s3" + secretsmanagerbackend "github.com/blackbirdworks/gopherstack/services/secretsmanager" ) // wireRDSIntegrations connects the RDS Data API and DMS to the services they read and write. @@ -53,6 +55,25 @@ func wireDMSStores(byName map[string]service.Registerable) { } dmsH.Backend.SetStores(stores) + + if h, hok := byName["SecretsManager"].(*secretsmanagerbackend.Handler); hok { + dmsH.Backend.SetSecrets(dmsSecrets{bk: h.Backend}) + } +} + +// dmsSecrets resolves endpoint SecretsManagerSecretId values against the in-process Secrets Manager. +type dmsSecrets struct { + bk secretsmanagerbackend.StorageBackend +} + +func (d dmsSecrets) SecretString(ctx context.Context, region, secretID string) (string, error) { + out, err := d.bk.GetSecretValue(secretsmanagerbackend.WithRegion(ctx, region), + &secretsmanagerbackend.GetSecretValueInput{SecretID: secretID}) + if err != nil { + return "", err + } + + return out.SecretString, nil } type dmsObjects struct{ bk *s3backend.InMemoryBackend } @@ -65,6 +86,42 @@ func (o *dmsObjects) PutObject(ctx context.Context, bucket, key string, body []b return err } +func (o *dmsObjects) ListObjects(ctx context.Context, bucket, prefix string) ([]string, error) { + var ( + keys []string + token *string + ) + + for { + out, err := o.bk.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), Prefix: aws.String(prefix), ContinuationToken: token, + }) + if err != nil { + return nil, err + } + + for _, obj := range out.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + + if !aws.ToBool(out.IsTruncated) || out.NextContinuationToken == nil { + return keys, nil + } + + token = out.NextContinuationToken + } +} + +func (o *dmsObjects) GetObject(ctx context.Context, bucket, key string) ([]byte, error) { + out, err := o.bk.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + if err != nil { + return nil, err + } + defer out.Body.Close() + + return io.ReadAll(out.Body) +} + func (o *dmsObjects) CheckBucket(ctx context.Context, bucket string) error { _, err := o.bk.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}) diff --git a/internal/app/dsql_wiring.go b/internal/app/dsql_wiring.go new file mode 100644 index 0000000000..2a96a602d0 --- /dev/null +++ b/internal/app/dsql_wiring.go @@ -0,0 +1,102 @@ +package app + +import ( + "net/http" + "path/filepath" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/dsql" + iambackend "github.com/blackbirdworks/gopherstack/services/iam" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +const ( + dsqlCAPath = "/_gopherstack/dsql/ca.pem" + dsqlClusterPath = "/_gopherstack/dsql/clusters/:id" +) + +// GetDSQLCADir persists the docker-mode DSQL CA under the data dir when --persist is on (dsql.CADirConfig). +func (c *CLI) GetDSQLCADir() string { + if !c.Persist { + return "" + } + + return filepath.Join(c.resolvedDataDir(), "dsql-ca") +} + +// wireDSQLAuth connects DSQL IAM-token authentication to IAM and STS and serves the data-plane CA and each +// cluster's connection address. +func wireDSQLAuth(e *echo.Echo, services []service.Registerable, enforceIAM bool) { + var ( + dsqlH *dsql.Handler + iamH *iambackend.Handler + stsH *stsbackend.Handler + ) + + for _, svc := range services { + switch h := svc.(type) { + case *dsql.Handler: + dsqlH = h + case *iambackend.Handler: + iamH = h + case *stsbackend.Handler: + stsH = h + } + } + + if dsqlH == nil { + return + } + + bk, ok := dsqlH.Backend.(*dsql.InMemoryBackend) + if !ok { + return + } + + bk.SetIAMAuth(func() dsql.IAMAuthConfig { return dsqlIAMAuthConfig(iamH, stsH, enforceIAM) }) + + e.GET(dsqlCAPath, func(c *echo.Context) error { + pem := bk.EngineCAPEM() + if len(pem) == 0 { + return c.String(http.StatusNotFound, "DSQL docker engine is not enabled") + } + + return c.Blob(http.StatusOK, "application/x-pem-file", pem) + }) + + e.GET(dsqlClusterPath, func(c *echo.Context) error { + id := c.Param("id") + + host, port, found := bk.ConnectAddr(id) + if !found { + return c.String(http.StatusNotFound, "cluster has no running data plane") + } + + cl, err := bk.GetCluster(id) + if err != nil { + return c.String(http.StatusNotFound, "cluster not found") + } + + return c.JSON(http.StatusOK, map[string]any{"host": host, "port": port, "endpoint": cl.Endpoint}) + }) +} + +func dsqlIAMAuthConfig(iamH *iambackend.Handler, stsH *stsbackend.Handler, enforceIAM bool) dsql.IAMAuthConfig { + cfg := dsql.IAMAuthConfig{Enforce: enforceIAM} + + if iamH != nil { + if ids, ok := iamH.Backend.(dsql.IAMAuthIdentityProvider); ok { + cfg.Identities = ids + } + } + + if stsH != nil { + if ss, ok := stsH.Backend.(dsql.IAMAuthSessionProvider); ok { + cfg.Sessions = ss + } + } + + return cfg +} diff --git a/internal/app/dsql_wiring_test.go b/internal/app/dsql_wiring_test.go new file mode 100644 index 0000000000..8cd7822e4f --- /dev/null +++ b/internal/app/dsql_wiring_test.go @@ -0,0 +1,123 @@ +package app + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/dsql" + iambackend "github.com/blackbirdworks/gopherstack/services/iam" + stsbackend "github.com/blackbirdworks/gopherstack/services/sts" +) + +type dsqlFakeInstance struct{} + +func (dsqlFakeInstance) Addr() (string, int) { return "127.0.0.1", 41234 } + +func (dsqlFakeInstance) Close(context.Context) {} + +type dsqlFakeLauncher struct{} + +func (dsqlFakeLauncher) Launch(context.Context, dsql.ClusterRef) (dsql.Instance, error) { + return dsqlFakeInstance{}, nil +} + +func TestWireDSQLAuth(t *testing.T) { + t.Parallel() + + b := dsql.NewInMemoryBackend() + b.EnableEngine(dsqlFakeLauncher{}) + t.Cleanup(b.Close) + + c, err := b.CreateCluster("123456789012", "us-east-1", dsql.CreateClusterInput{}) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, _, ok := b.ConnectAddr(c.Identifier) + + return ok + }, 10*time.Second, time.Millisecond) + + e := echo.New() + services := []service.Registerable{ + dsql.NewHandler(b), iambackend.NewHandler(iambackend.NewInMemoryBackend()), + stsbackend.NewHandler(stsbackend.NewInMemoryBackend()), + } + wireDSQLAuth(e, services, true) + + tests := []struct { + name string + path string + code int + }{ + {name: "cluster address", path: "/_gopherstack/dsql/clusters/" + c.Identifier, code: http.StatusOK}, + {name: "unknown cluster", path: "/_gopherstack/dsql/clusters/nope", code: http.StatusNotFound}, + {name: "no ca without docker engine", path: dsqlCAPath, code: http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, tt.path, nil)) + assert.Equal(t, tt.code, rec.Code) + + if tt.code != http.StatusOK { + return + } + + var got struct { + Host string `json:"host"` + Endpoint string `json:"endpoint"` + Port int `json:"port"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + assert.Equal(t, "127.0.0.1", got.Host) + assert.Equal(t, 41234, got.Port) + assert.Equal(t, c.Endpoint, got.Endpoint) + }) + } +} + +func TestDSQLIAMAuthConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + withIAM bool + enforce bool + }{ + {name: "iam and sts present", withIAM: true, enforce: true}, + {name: "services absent"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var ( + iamH *iambackend.Handler + stsH *stsbackend.Handler + ) + + if tt.withIAM { + iamH = iambackend.NewHandler(iambackend.NewInMemoryBackend()) + stsH = stsbackend.NewHandler(stsbackend.NewInMemoryBackend()) + } + + cfg := dsqlIAMAuthConfig(iamH, stsH, tt.enforce) + assert.Equal(t, tt.enforce, cfg.Enforce) + assert.Equal(t, tt.withIAM, cfg.Identities != nil) + assert.Equal(t, tt.withIAM, cfg.Sessions != nil) + }) + } +} diff --git a/internal/app/engines.go b/internal/app/engines.go index 85d0bb7837..4a6847cf58 100644 --- a/internal/app/engines.go +++ b/internal/app/engines.go @@ -67,6 +67,10 @@ func engineEntries() []engineEntry { "oci-oke-engine", "OKE_ENGINE", enginesStub, }, {func(c *CLI) *string { return &c.OCIMySQL.Engine }, "oci-mysql-engine", "OCI_MYSQL_ENGINE", enginesStub}, + { + func(c *CLI) *string { return &c.OCIDatabase.Engine }, + "oci-database-engine", "OCI_DATABASE_ENGINE", enginesStub, + }, {func(c *CLI) *string { return &c.RDSEngine }, "rds-engine", "RDS_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.DMSEngine }, "dms-engine", "DMS_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.DocDBEngine }, "docdb-engine", "DOCDB_ENGINE", enginesStub}, @@ -80,6 +84,9 @@ func engineEntries() []engineEntry { {func(c *CLI) *string { return &c.SageMakerEngine }, "sagemaker-engine", "SAGEMAKER_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.CodeBuild.Engine }, "codebuild-engine", "CODEBUILD_ENGINE", enginesStub}, {func(c *CLI) *string { return &c.Glue.Engine }, "glue-engine", "GLUE_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.DSQL.Engine }, "dsql-engine", "DSQL_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.LambdaWeb.Engine }, "lambdaweb-engine", "LAMBDAWEB_ENGINE", enginesStub}, + {func(c *CLI) *string { return &c.Transcribe.Engine }, "transcribe-engine", "TRANSCRIBE_ENGINE", enginesStub}, } } diff --git a/internal/app/gcpcloudbuild_wiring.go b/internal/app/gcpcloudbuild_wiring.go index 30d91c63f0..bd0b718ff7 100644 --- a/internal/app/gcpcloudbuild_wiring.go +++ b/internal/app/gcpcloudbuild_wiring.go @@ -106,6 +106,7 @@ func wireGCPStorageConsumers(byName map[string]service.Registerable) { wireGCPContainer(byName) wireGCPIdentityToolkit(byName) wireGCPBatch(byName) + wireGCPStorageSecurity(byName) } // wireGCPCloudBuild connects Cloud Build to Cloud Storage, Pub/Sub, Cloud Logging, Secret Manager and the diff --git a/internal/app/gcpstorage_wiring.go b/internal/app/gcpstorage_wiring.go new file mode 100644 index 0000000000..a96591e116 --- /dev/null +++ b/internal/app/gcpstorage_wiring.go @@ -0,0 +1,45 @@ +package app + +import ( + "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + gcpiambackend "github.com/blackbirdworks/gopherstack/services/gcpiam" + gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" + gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" +) + +type gcsPubSubNotifier struct { + ps *gcppubsubbackend.InMemoryBackend +} + +func (n gcsPubSubNotifier) TopicExists(topic string) bool { + _, err := n.ps.GetTopic(topic) + + return err == nil +} + +func (n gcsPubSubNotifier) Publish(topic string, data []byte, attrs map[string]string) error { + _, err := n.ps.Publish(&pubsubpb.PublishRequest{ + Topic: topic, Messages: []*pubsubpb.PubsubMessage{{Data: data, Attributes: attrs}}, + }) + + return err +} + +// wireGCPStorageSecurity gives Cloud Storage Pub/Sub delivery for bucket notifications and Cloud IAM service +// account keys for verifying signed URLs. +func wireGCPStorageSecurity(byName map[string]service.Registerable) { + gcs, ok := byName["GCPStorage"].(*gcpstoragebackend.Handler) + if !ok { + return + } + + if ps, isPS := byName["GCPPubSub"].(*gcppubsubbackend.Handler); isPS { + gcs.Backend.SetNotifier(gcsPubSubNotifier{ps: ps.Backend}) + } + + if iam, isIAM := byName["GCPIAM"].(*gcpiambackend.Handler); isIAM { + gcs.Backend.SetKeyResolver(iam.Backend.VerificationKeys) + } +} diff --git a/internal/app/gcpstorage_wiring_test.go b/internal/app/gcpstorage_wiring_test.go new file mode 100644 index 0000000000..ac8e976c93 --- /dev/null +++ b/internal/app/gcpstorage_wiring_test.go @@ -0,0 +1,64 @@ +package app + +import ( + "context" + "testing" + + "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + gcppubsubbackend "github.com/blackbirdworks/gopherstack/services/gcppubsub" + gcpstoragebackend "github.com/blackbirdworks/gopherstack/services/gcpstorage" +) + +func TestGCPStorageNotificationsDeliverToPubSub(t *testing.T) { + t.Parallel() + + gcs := gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()) + ps := gcppubsubbackend.NewHandler(gcppubsubbackend.NewInMemoryBackend()) + byName := map[string]service.Registerable{"GCPStorage": gcs, "GCPPubSub": ps} + + wireGCPStorageSecurity(byName) + + const ( + topic = "projects/p/topics/events" + sub = "projects/p/subscriptions/events-sub" + ) + + _, err := ps.Backend.CreateTopic(&pubsubpb.Topic{Name: topic}) + require.NoError(t, err) + + _, err = ps.Backend.CreateSubscription(&pubsubpb.Subscription{Name: sub, Topic: topic}) + require.NoError(t, err) + + _, err = gcs.Backend.CreateBucket("p", gcpstoragebackend.Patch{"name": []byte(`"bkt"`)}) + require.NoError(t, err) + + _, err = gcs.Backend.AddNotification("bkt", gcpstoragebackend.Notification{ + Topic: "//pubsub.googleapis.com/" + topic, PayloadFormat: "JSON_API_V1", + }) + require.NoError(t, err) + + _, err = gcs.Backend.PutObject("bkt", "k", []byte("v"), gcpstoragebackend.Patch{}, gcpstoragebackend.Preconds{}) + require.NoError(t, err) + + res, err := ps.SubscriberAPI().Pull(context.Background(), &pubsubpb.PullRequest{ + Subscription: sub, MaxMessages: 10, ReturnImmediately: true, //nolint:staticcheck // immediate poll in tests + }) + require.NoError(t, err) + require.Len(t, res.GetReceivedMessages(), 1) + assert.Equal(t, "OBJECT_FINALIZE", res.GetReceivedMessages()[0].GetMessage().GetAttributes()["eventType"]) +} + +func TestGCPStorageSecurityWiringWithoutPeers(t *testing.T) { + t.Parallel() + + assert.NotPanics(t, func() { wireGCPStorageSecurity(map[string]service.Registerable{}) }) + assert.NotPanics(t, func() { + wireGCPStorageSecurity(map[string]service.Registerable{ + "GCPStorage": gcpstoragebackend.NewHandler(gcpstoragebackend.NewInMemoryBackend()), + }) + }) +} diff --git a/internal/app/lambda_runtime_env.go b/internal/app/lambda_runtime_env.go index 6d6152ad46..2f57ef9fc3 100644 --- a/internal/app/lambda_runtime_env.go +++ b/internal/app/lambda_runtime_env.go @@ -57,6 +57,8 @@ func lambdaEndpointURL(dockerHost, port string) string { // wireLambdaRuntimeEnv gives function containers the emulator endpoint and // execution-role credentials. func wireLambdaRuntimeEnv(cli *CLI, byName map[string]service.Registerable) { + wireLambdaWeb(cli, byName) + lambdaH, ok := byName["Lambda"].(*lambdabackend.Handler) if !ok { return diff --git a/internal/app/lambdaweb_wiring.go b/internal/app/lambdaweb_wiring.go new file mode 100644 index 0000000000..19e7351cd5 --- /dev/null +++ b/internal/app/lambdaweb_wiring.go @@ -0,0 +1,117 @@ +package app + +import ( + "context" + "errors" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + cwlogsbackend "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" + s3backend "github.com/blackbirdworks/gopherstack/services/s3" +) + +// lambdaWebCode serves web function deployment packages from versioned S3 buckets. +type lambdaWebCode struct { + backend s3backend.StorageBackend +} + +func (c lambdaWebCode) ResolveObject(ctx context.Context, bucket, key, versionID string) (string, error) { + ver, err := c.backend.GetBucketVersioning(ctx, &awss3.GetBucketVersioningInput{Bucket: aws.String(bucket)}) + if err != nil { + if errors.Is(err, s3backend.ErrNoSuchBucket) { + return "", lambdawebbackend.ErrBucketNotFound + } + + return "", err + } + + if ver.Status != s3types.BucketVersioningStatusEnabled { + return "", lambdawebbackend.ErrBucketNotVersioned + } + + in := &awss3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)} + if versionID != "" { + in.VersionId = aws.String(versionID) + } + + head, err := c.backend.HeadObject(ctx, in) + if err != nil { + if errors.Is(err, s3backend.ErrNoSuchKey) || errors.Is(err, s3backend.ErrNoSuchBucket) { + return "", lambdawebbackend.ErrObjectNotFound + } + + return "", err + } + + return aws.ToString(head.VersionId), nil +} + +func (c lambdaWebCode) GetObject(ctx context.Context, bucket, key, versionID string) ([]byte, error) { + in := &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)} + if versionID != "" { + in.VersionId = aws.String(versionID) + } + + out, err := c.backend.GetObject(ctx, in) + if err != nil { + return nil, err + } + defer out.Body.Close() + + return io.ReadAll(out.Body) +} + +// lambdaWebCredentials adapts the Lambda STS role-credential issuer to lambdaweb.CredentialIssuer. +type lambdaWebCredentials struct { + inner *lambdaRoleCredentials +} + +func (a lambdaWebCredentials) IssueCredentials(roleARN, sessionName string) (lambdawebbackend.Credentials, error) { + c, err := a.inner.IssueFunctionCredentials(roleARN, sessionName) + if err != nil { + return lambdawebbackend.Credentials{}, err + } + + return lambdawebbackend.Credentials{ + AccessKeyID: c.AccessKeyID, SecretAccessKey: c.SecretAccessKey, SessionToken: c.SessionToken, + }, nil +} + +// wireLambdaWeb gives Lambda Web Functions its S3 code store, CloudWatch Logs sink, execution-role +// credentials, emulator endpoint and the SigV4 secret IamAuth endpoints verify against. +func wireLambdaWeb(cli *CLI, byName map[string]service.Registerable) { + h, ok := byName["LambdaWeb"].(*lambdawebbackend.Handler) + if !ok { + return + } + + bk, ok := h.Backend.(*lambdawebbackend.InMemoryBackend) + if !ok { + return + } + + if s3H, s3Ok := byName["S3"].(*s3backend.S3Handler); s3Ok { + bk.SetCodeStore(lambdaWebCode{backend: s3H.Backend}) + } + + if logsH, logsOk := byName["CloudWatchLogs"].(*cwlogsbackend.Handler); logsOk { + if logsBk, bkOk := logsH.Backend.(*cwlogsbackend.InMemoryBackend); bkOk { + bk.SetLogSink(codebuildLogs{backend: logsBk}) + } + } + + bk.SetCredentialIssuer(lambdaWebCredentials{inner: &lambdaRoleCredentials{byName: byName}}) + + if cli != nil { + bk.SetSigV4Secret(cli.SigV4Secret) + + if cli.Port != "" { + bk.SetEndpointURL(lambdaEndpointURL(cli.Lambda.DockerHost, cli.Port)) + } + } +} diff --git a/internal/app/lifecycle.go b/internal/app/lifecycle.go index 7a5a3e436b..5257b1f62f 100644 --- a/internal/app/lifecycle.go +++ b/internal/app/lifecycle.go @@ -50,6 +50,7 @@ import ( kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" kinesisanalyticsv2backend "github.com/blackbirdworks/gopherstack/services/kinesisanalyticsv2" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + lambdawebbackend "github.com/blackbirdworks/gopherstack/services/lambdaweb" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" managedblockchainbackend "github.com/blackbirdworks/gopherstack/services/managedblockchain" medialivebackend "github.com/blackbirdworks/gopherstack/services/medialive" @@ -253,6 +254,14 @@ func applyFIS(reg service.Registerable, l cliconfig.LifecycleSettings) { } } +func applyLambdaWeb(reg service.Registerable, l cliconfig.LifecycleSettings) { + if h, ok := reg.(*lambdawebbackend.Handler); ok { + if bk, isBk := h.Backend.(*lambdawebbackend.InMemoryBackend); isBk { + bk.SetLifecycleDelay(l.Effective(l.LambdaWeb)) + } + } +} + func applyDSQL(reg service.Registerable, l cliconfig.LifecycleSettings) { if h, ok := reg.(*dsqlbackend.Handler); ok { if bk, isBk := h.Backend.(*dsqlbackend.InMemoryBackend); isBk { @@ -780,6 +789,7 @@ func wireLifecycleDelays(byName map[string]service.Registerable, l cliconfig.Lif "AppStream": applyAppStream, "FIS": applyFIS, "DSQL": applyDSQL, + "LambdaWeb": applyLambdaWeb, "GlobalAccelerator": applyGlobalAccelerator, "CodeGuruReviewer": applyCodeGuruReviewer, "Connect": applyConnect, diff --git a/internal/app/lifecycle_cmd.go b/internal/app/lifecycle_cmd.go index 090fa06b9c..0b4e5234d6 100644 --- a/internal/app/lifecycle_cmd.go +++ b/internal/app/lifecycle_cmd.go @@ -27,6 +27,7 @@ const ( lifecycleDefaultTail = 100 lifecycleLogTailMaxByte = 1 << 20 lifecycleSocketInside = "/var/run/docker.sock" + lifecycleRootUser = "0:0" // image default 65532 can't write a host bind mount or open the socket lifecycleSchemeHTTPS = "https" lifecycleStopGrace = 2 * lifecycleStopTimeout localstackAvailable = "available" diff --git a/internal/app/lifecycle_cmd_test.go b/internal/app/lifecycle_cmd_test.go index a943b50c5a..25c309ce23 100644 --- a/internal/app/lifecycle_cmd_test.go +++ b/internal/app/lifecycle_cmd_test.go @@ -319,10 +319,12 @@ func TestLifecycleStartCreatesContainer(t *testing.T) { assert.Equal(t, wantPorts, spec.Ports) if tt.mount { + assert.Equal(t, "0:0", spec.User) require.Len(t, spec.Mounts, 1) assert.True(t, strings.HasSuffix(spec.Mounts[0], ":/data")) assert.True(t, strings.HasPrefix(spec.Mounts[0], filepath.Join(dir, "data-n1"))) } else { + assert.Empty(t, spec.User) assert.Empty(t, spec.Mounts) } diff --git a/internal/app/lifecycle_foreground_test.go b/internal/app/lifecycle_foreground_test.go index b9d30e8e1a..8132969dca 100644 --- a/internal/app/lifecycle_foreground_test.go +++ b/internal/app/lifecycle_foreground_test.go @@ -261,6 +261,7 @@ func TestLifecycleDockerSocketMount(t *testing.T) { } assert.Contains(t, spec.Mounts, tt.wantMount) + assert.Equal(t, "0:0", spec.User) assert.Contains(t, spec.Env, "DOCKER_HOST=unix:///var/run/docker.sock") assert.Contains(t, out.String(), "root-equivalent") }) diff --git a/internal/app/lifecycle_run.go b/internal/app/lifecycle_run.go index 9ba2fe5363..18ed776a28 100644 --- a/internal/app/lifecycle_run.go +++ b/internal/app/lifecycle_run.go @@ -246,11 +246,13 @@ func (c *StartCmd) runContainer(ctx context.Context, d *lifecycleDeps, endpoint, if dataDir != "" { env = append(env, "GOPHERSTACK_DATA_DIR="+lifecycleContainerData) spec.Mounts = []string{dataDir + ":" + lifecycleContainerData} + spec.User = lifecycleRootUser } if c.DockerSocket { sock := c.dockerSocketPath(d.Getenv) spec.Mounts = append(spec.Mounts, sock+":"+lifecycleSocketInside) + spec.User = lifecycleRootUser env = append(env, "DOCKER_HOST=unix://"+lifecycleSocketInside) fmt.Fprintf(d.Err, "warning: mounting %s gives the server root-equivalent control of the host Docker daemon\n", sock) diff --git a/internal/app/localstack_compat_test.go b/internal/app/localstack_compat_test.go index 1d0715c54b..71fd0ea8b0 100644 --- a/internal/app/localstack_compat_test.go +++ b/internal/app/localstack_compat_test.go @@ -223,6 +223,18 @@ func TestLocalstackEnvPlan(t *testing.T) { "GOPHERSTACK_SQS_DISABLE_MESSAGE_RETENTION": "true", }, }, + { + name: "enable_dms_runs_the_engine", environ: []string{"ENABLE_DMS=1"}, + want: map[string]string{"DMS_ENGINE": "docker"}, + }, + { + name: "enable_dms_off", environ: []string{"ENABLE_DMS=0"}, + want: map[string]string{}, + }, + { + name: "enable_dms_keeps_explicit_engine", environ: []string{"ENABLE_DMS=1", "DMS_ENGINE=stub"}, + want: map[string]string{}, + }, { name: "sqs_toggles_on_noop", environ: []string{"SQS_DELAY_PURGE_RETRY=1", "SQS_ENABLE_MESSAGE_RETENTION_PERIOD=1"}, diff --git a/internal/app/localstack_options.go b/internal/app/localstack_options.go index daf8b14c1b..d54aa78d94 100644 --- a/internal/app/localstack_options.go +++ b/internal/app/localstack_options.go @@ -159,6 +159,7 @@ func (p *envPlan) planOptions() { p.planPortRange() p.planSQS() + p.planDMS() p.planLimits() p.planS3Signatures() @@ -183,6 +184,13 @@ func (p *envPlan) planOptions() { } } +// planDMS maps ENABLE_DMS=1, which LocalStack documents as activating DMS, onto the engine that moves data. +func (p *envPlan) planDMS() { + if p.truthy("ENABLE_DMS") { + p.put("DMS_ENGINE", "docker") + } +} + // planSQS inverts LocalStack's positive SQS toggles onto the negative native switches. func (p *envPlan) planSQS() { for _, m := range []struct{ ls, native string }{ diff --git a/internal/app/ocidatabase_wiring.go b/internal/app/ocidatabase_wiring.go new file mode 100644 index 0000000000..36e928c499 --- /dev/null +++ b/internal/app/ocidatabase_wiring.go @@ -0,0 +1,37 @@ +package app + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +// GetOCIDatabaseSettings returns OCI Database settings (ocidatabase.ConfigProvider). +func (c *CLI) GetOCIDatabaseSettings() ocidatabase.Settings { return c.OCIDatabase } + +// wireOCIDatabase shares OCI Identity keys and compartments and checks private endpoint subnets in OCI Core. +func wireOCIDatabase(byName map[string]service.Registerable) { + h, ok := byName["OCIDatabase"].(*ocidatabase.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + h.Backend.Auth.Link(identityH.Backend.Auth) + h.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + h.Backend.SetSubnetResolver(func(id string) (ocidatabase.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocidatabase.SubnetInfo{CIDR: ref.CidrBlock}, live + }) + } +} + +// GetOCIDatabaseHandler returns the OCIDatabase handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIDatabaseHandler() service.Registerable { return c.ociDatabaseHandler } diff --git a/internal/app/ocidatabase_wiring_test.go b/internal/app/ocidatabase_wiring_test.go new file mode 100644 index 0000000000..fe4da73e1c --- /dev/null +++ b/internal/app/ocidatabase_wiring_test.go @@ -0,0 +1,103 @@ +package app + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidatabase" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIDatabase(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8170, ocidatabase.DefaultPort) + + tests := []struct { + name string + rangeStart int + rangeEnd int + wantTaken bool + }{ + {"port inside pool range", 8100, 8200, true}, + {"port outside pool range", 10000, 10100, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) + require.NoError(t, err) + + cli := CLI{OCIDatabase: ocidatabase.DefaultSettings()} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocidatabase.DefaultPort)) + }) + } +} + +func TestOCIDatabaseEngineFollowsGlobalSwitch(t *testing.T) { + t.Parallel() + + tests := []struct { + cli func() *CLI + name string + want string + }{ + { + name: "stub-default", want: ocidatabase.EngineStub, + cli: func() *CLI { return &CLI{OCIDatabase: ocidatabase.DefaultSettings()} }, + }, + { + name: "explicit-docker", want: ocidatabase.EngineDocker, + cli: func() *CLI { return &CLI{OCIDatabase: ocidatabase.Settings{Engine: ocidatabase.EngineDocker}} }, + }, + { + name: "engines-switch", want: ocidatabase.EngineDocker, + cli: func() *CLI { + return &CLI{ + OCIDatabase: ocidatabase.DefaultSettings(), + engineModes: map[string]string{"oci-database-engine": ocidatabase.EngineDocker}, + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, tt.cli().GetOCIDatabaseEngine()) + }) + } +} + +func TestWireOCIDatabaseAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + dbH := ocidatabase.NewHandler(ocidatabase.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIDatabase(map[string]service.Registerable{"OCIDatabase": dbH}) }) + assert.NotPanics(t, func() { wireOCIDatabase(map[string]service.Registerable{}) }) + + wireOCIDatabase(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIDatabase": dbH}) + + _, err := dbH.Backend.ListBackups( + ocidatabase.BackupFilter{Compartment: "ocid1.compartment.oc1..ghost"}, + ocidatabase.ListOpts{}, + ) + require.Error(t, err, "compartments are checked against OCI Identity") + + cli := CLI{OCIDatabase: ocidatabase.Settings{Port: 8170}} + assert.Equal(t, 8170, cli.GetOCIDatabaseSettings().Port) +} diff --git a/internal/app/ocidns_wiring.go b/internal/app/ocidns_wiring.go new file mode 100644 index 0000000000..6ddd8ffa34 --- /dev/null +++ b/internal/app/ocidns_wiring.go @@ -0,0 +1,49 @@ +package app + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidns" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +// GetOCIDNSSettings returns OCI DNS settings (ocidns.ConfigProvider). +func (c *CLI) GetOCIDNSSettings() ocidns.Settings { return c.OCIDNS } + +// wireOCIDNS shares OCI Identity keys and compartments and attaches private resolvers to OCI Core VCNs. +func wireOCIDNS(byName map[string]service.Registerable) { + dnsH, ok := byName["OCIDNS"].(*ocidns.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + dnsH.Backend.Auth.Link(identityH.Backend.Auth) + dnsH.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + dnsH.Backend.SetVcnSource(func() []ocidns.VcnInfo { + vcns := coreH.Backend.ListVcns() + out := make([]ocidns.VcnInfo, 0, len(vcns)) + + for _, v := range vcns { + out = append(out, ocidns.VcnInfo{ + ID: v.ID, CompartmentID: v.CompartmentID, DisplayName: v.DisplayName, Created: v.TimeCreated, + }) + } + + return out + }) + dnsH.Backend.SetSubnetResolver(func(id string) (ocidns.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocidns.SubnetInfo{VcnID: ref.VcnID, Cidr: ref.CidrBlock}, live + }) + } +} + +// GetOCIDNSHandler returns the OCIDNS handler for the dashboard proxy. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIDNSHandler() service.Registerable { return c.ociDNSHandler } diff --git a/internal/app/ocidns_wiring_test.go b/internal/app/ocidns_wiring_test.go new file mode 100644 index 0000000000..c3735fe8fb --- /dev/null +++ b/internal/app/ocidns_wiring_test.go @@ -0,0 +1,110 @@ +package app + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocidns" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIDNS(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8168, ocidns.DefaultPort) + assert.Equal(t, 8169, ocidns.DefaultDNSPort) + + tests := []struct { + name string + rangeStart int + rangeEnd int + wantTaken bool + }{ + {"ports inside pool range", 8100, 8200, true}, + {"ports outside pool range", 10000, 10100, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) + require.NoError(t, err) + + cli := CLI{OCIDNS: ocidns.DefaultSettings()} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocidns.DefaultPort)) + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocidns.DefaultDNSPort)) + }) + } +} + +func TestWireOCIDNSAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + dnsH := ocidns.NewHandler(ocidns.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIDNS(map[string]service.Registerable{"OCIDNS": dnsH}) }) + assert.NotPanics(t, func() { wireOCIDNS(map[string]service.Registerable{}) }) + + wireOCIDNS(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIDNS": dnsH}) + + _, _, err := dnsH.Backend.CreateView(ocidns.CreateViewIn{CompartmentID: "ocid1.compartment.oc1..ghost"}) + require.Error(t, err, "compartments are checked against OCI Identity") + + const compartment = "ocid1.compartment.oc1..c" + + vcn, _, aerr := coreH.Backend.CreateVcn(ocicorebackend.CreateVcnIn{ + CompartmentID: compartment, CidrBlocks: []string{"10.0.0.0/16"}, DisplayName: "core-vcn", + }) + require.Nil(t, aerr) + + vcnID := vcn.(*ocicorebackend.Vcn).ID //nolint:forcetypeassert // CreateVcn returns *Vcn + + sub, _, aerr := coreH.Backend.CreateSubnet(ocicorebackend.CreateSubnetIn{ + CompartmentID: compartment, VcnID: vcnID, CidrBlock: "10.0.1.0/24", + }) + require.Nil(t, aerr) + + subID := sub.(*ocicorebackend.Subnet).ID //nolint:forcetypeassert // CreateSubnet returns *Subnet + + dnsH.Backend.SyncVcns() + + got, err := dnsH.Backend.ListResolvers(ocidns.ResolverFilter{}, ocidns.ListOpts{}) + require.Error(t, err, "compartmentId is required") + assert.Empty(t, got.Items) + + dnsH.Backend.SetCompartmentChecker(nil) + + got, err = dnsH.Backend.ListResolvers(ocidns.ResolverFilter{Compart: compartment}, ocidns.ListOpts{}) + require.NoError(t, err) + require.Len(t, got.Items, 1, "every OCI Core VCN gets a resolver") + assert.Equal(t, vcnID, got.Items[0].AttachedVcnID) + assert.Equal(t, "core-vcn", got.Items[0].DisplayName) + + yes, no := true, false + + ep, _, err := dnsH.Backend.CreateEndpoint(got.Items[0].ID, ocidns.CreateEndpointIn{ + Name: "ep", SubnetID: subID, IsForwarding: &yes, IsListening: &no, + }) + require.NoError(t, err, "subnets resolve through OCI Core") + assert.Equal(t, "10.0.1.2", *ep.ForwardingAddress) + + _, _, err = dnsH.Backend.CreateEndpoint(got.Items[0].ID, ocidns.CreateEndpointIn{ + Name: "ghost", SubnetID: "ocid1.subnet.oc1..ghost", IsForwarding: &yes, IsListening: &no, + }) + require.Error(t, err) + + cli := CLI{OCIDNS: ocidns.Settings{Port: 8168, DNSPort: 8169}} + assert.Equal(t, 8168, cli.GetOCIDNSSettings().Port) + assert.Equal(t, 8169, cli.GetOCIDNSSettings().DNSPort) +} diff --git a/internal/app/ocifilestorage_wiring.go b/internal/app/ocifilestorage_wiring.go new file mode 100644 index 0000000000..248cf8b3b9 --- /dev/null +++ b/internal/app/ocifilestorage_wiring.go @@ -0,0 +1,85 @@ +package app + +import ( + "encoding/base64" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" + ocivaultbackend "github.com/blackbirdworks/gopherstack/services/ocivault" +) + +// GetOCIFileStorageSettings returns OCI File Storage settings (ocifilestorage.ConfigProvider). +func (c *CLI) GetOCIFileStorageSettings() ocifilestorage.Settings { return c.OCIFileStorage } + +// GetOCIFileStorageEngine returns the OCI File Storage engine mode (ocifilestorage.EngineConfigProvider). The NFS +// engine needs no container runtime, so the global --engines switch does not turn it on. +func (c *CLI) GetOCIFileStorageEngine() string { return c.OCIFileStorage.Engine } + +// GetOCIFileStorageHandler returns the OCIFileStorage handler. +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetOCIFileStorageHandler() service.Registerable { return c.ociFileStorageHandler } + +// wireOCIFileStorage shares OCI Identity keys and compartments, backs mount targets with OCI Core VNICs and reads +// Kerberos keytabs and LDAP passwords from OCI Vault secrets. +func wireOCIFileStorage(byName map[string]service.Registerable) { + h, ok := byName["OCIFileStorage"].(*ocifilestorage.Handler) + if !ok { + return + } + + if identityH, found := byName["OCIIdentity"].(*ociidentitybackend.Handler); found { + h.Backend.Auth.Link(identityH.Backend.Auth) + h.Backend.SetCompartmentChecker(identityH.Backend.CompartmentActive) + } + + if coreH, found := byName["OCICore"].(*ocicorebackend.Handler); found { + h.Backend.SetNetwork(ocifilestorage.Network{ + Subnet: func(id string) (ocifilestorage.SubnetInfo, bool) { + ref, live := coreH.Backend.LookupSubnet(id) + + return ocifilestorage.SubnetInfo{ + VcnID: ref.VcnID, CIDR: ref.CidrBlock, AD: ref.AvailabilityDomain, CompartmentID: ref.CompartmentID, + }, live + }, + CreateNic: func(r ocifilestorage.NicRequest) (ocifilestorage.Nic, error) { + nic, err := coreH.Backend.CreateServiceNic(ocicorebackend.ServiceNicIn{ + CompartmentID: r.CompartmentID, AD: r.AD, SubnetID: r.SubnetID, DisplayName: r.DisplayName, + HostnameLabel: r.Hostname, PrivateIP: r.IPAddress, NsgIDs: r.NsgIDs, + }) + if err != nil { + return ocifilestorage.Nic{}, err + } + + return ocifilestorage.Nic{ + VnicID: nic.VnicID, + PrivateIPID: nic.PrivateIPID, + IPAddress: nic.IPAddress, + }, nil + }, + DeleteNic: coreH.Backend.DeleteServiceNic, + }) + } + + if vaultH, found := byName["OCIVault"].(*ocivaultbackend.Handler); found { + h.Backend.SetSecretLookup(func(id string, version *int) ([]byte, bool) { + sel := ocivaultbackend.BundleSelector{Stage: "CURRENT"} + + if version != nil { + v := int64(*version) + sel = ocivaultbackend.BundleSelector{VersionNumber: &v} + } + + bundle, _, err := vaultH.Backend.GetSecretBundle(id, sel) + if err != nil || bundle.Content == nil { + return nil, false + } + + raw, derr := base64.StdEncoding.DecodeString(bundle.Content.Content) + + return raw, derr == nil + }) + } +} diff --git a/internal/app/ocifilestorage_wiring_test.go b/internal/app/ocifilestorage_wiring_test.go new file mode 100644 index 0000000000..f26e9a2991 --- /dev/null +++ b/internal/app/ocifilestorage_wiring_test.go @@ -0,0 +1,149 @@ +package app + +import ( + "encoding/json" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/service" + ocicorebackend "github.com/blackbirdworks/gopherstack/services/ocicore" + "github.com/blackbirdworks/gopherstack/services/ocifilestorage" + ociidentitybackend "github.com/blackbirdworks/gopherstack/services/ociidentity" +) + +func TestReserveFixedServicePorts_OCIFileStorage(t *testing.T) { + t.Parallel() + + assert.Equal(t, 8171, ocifilestorage.DefaultPort) + + tests := []struct { + name string + rangeStart int + rangeEnd int + wantTaken bool + }{ + {"port inside pool range", 8100, 8200, true}, + {"port outside pool range", 10000, 10100, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + alloc, err := portalloc.New(tt.rangeStart, tt.rangeEnd) + require.NoError(t, err) + + cli := CLI{OCIFileStorage: ocifilestorage.DefaultSettings()} + reserveFixedServicePorts(t.Context(), slog.Default(), alloc, cli) + + assert.Equal(t, tt.wantTaken, alloc.IsAllocated(ocifilestorage.DefaultPort)) + }) + } +} + +func TestOCIFileStorageEngineIgnoresGlobalSwitch(t *testing.T) { + t.Parallel() + + tests := []struct { + cli *CLI + name string + want string + }{ + {name: "default", want: ocifilestorage.EngineStub, cli: &CLI{OCIFileStorage: ocifilestorage.DefaultSettings()}}, + { + name: "explicit", + want: ocifilestorage.EngineNFS, + cli: &CLI{OCIFileStorage: ocifilestorage.Settings{Engine: ocifilestorage.EngineNFS}}, + }, + { + name: "global-docker", want: ocifilestorage.EngineStub, + cli: &CLI{ + OCIFileStorage: ocifilestorage.DefaultSettings(), + engineModes: map[string]string{"oci-filestorage-engine": "docker"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, tt.cli.GetOCIFileStorageEngine()) + }) + } +} + +func idOf(t *testing.T, v any) string { + t.Helper() + + raw, err := json.Marshal(v) + require.NoError(t, err) + + var out struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(raw, &out)) + + return out.ID +} + +func TestWireOCIFileStorageAgainstCoreAndIdentity(t *testing.T) { + t.Parallel() + + idH := ociidentitybackend.NewHandler(ociidentitybackend.NewInMemoryBackend()) + coreH := ocicorebackend.NewHandler(ocicorebackend.NewInMemoryBackend()) + fsH := ocifilestorage.NewHandler(ocifilestorage.NewInMemoryBackend()) + + assert.NotPanics(t, func() { wireOCIFileStorage(map[string]service.Registerable{"OCIFileStorage": fsH}) }) + assert.NotPanics(t, func() { wireOCIFileStorage(map[string]service.Registerable{}) }) + + wireOCIFileStorage(map[string]service.Registerable{"OCICore": coreH, "OCIFileStorage": fsH}) + + const comp = "ocid1.compartment.oc1..c" + + vcn, _, aerr := coreH.Backend.CreateVcn( + ocicorebackend.CreateVcnIn{CompartmentID: comp, DisplayName: "v", CidrBlock: "10.0.0.0/16"}, + ) + require.Nil(t, aerr) + + sub, _, aerr := coreH.Backend.CreateSubnet( + ocicorebackend.CreateSubnetIn{CompartmentID: comp, VcnID: idOf(t, vcn), CidrBlock: "10.0.1.0/24"}, + ) + require.Nil(t, aerr) + + create := func() error { + _, _, err := fsH.Backend.CreateMountTarget(ocifilestorage.CreateMountTargetIn{ + AvailabilityDomain: "ad-1", CompartmentID: comp, SubnetID: idOf(t, sub), IPAddress: new("10.0.1.40"), + }, "") + + return err + } + + require.NoError(t, create()) + require.Error(t, create(), "the mount target holds a real private IP in OCI Core") + + mts, err := fsH.Backend.ListMountTargets( + ocifilestorage.MTFilter{Compartment: comp, AD: "ad-1"}, + ocifilestorage.ListOpts{}, + ) + require.NoError(t, err) + require.Len(t, mts.Data, 1) + + require.NoError(t, fsH.Backend.DeleteMountTarget(mts.Data[0]["id"].(string), "", false)) + require.NoError(t, create(), "deleting the mount target releases its VNIC") + + wireOCIFileStorage(map[string]service.Registerable{"OCIIdentity": idH, "OCICore": coreH, "OCIFileStorage": fsH}) + + _, err = fsH.Backend.ListFileSystems( + ocifilestorage.FSFilter{Compartment: "ocid1.compartment.oc1..ghost", AD: "ad-1"}, ocifilestorage.ListOpts{}, + ) + require.Error(t, err, "compartments are checked against OCI Identity") + + cli := CLI{OCIFileStorage: ocifilestorage.Settings{Port: 8171}} + assert.Equal(t, 8171, cli.GetOCIFileStorageSettings().Port) +} diff --git a/internal/app/routing_equivalence_test.go b/internal/app/routing_equivalence_test.go index d1637dc61b..f4528f3b16 100644 --- a/internal/app/routing_equivalence_test.go +++ b/internal/app/routing_equivalence_test.go @@ -254,7 +254,8 @@ func TestRoutingCorpusCoversServices(t *testing.T) { "AzureContainerRegistry", "AzureFunctions", "GCPArtifactRegistry", "OCIObjectStorage", "OCIVault", "OCIStreaming", "OCIIdentity", "OCIFunctions", "OCICore", "OCIQueue", "OCINotifications", "OCIContainerEngine", "OCIMySQL", "OCILogging", "OCIMonitoring", "OCINoSQL", "OCIAPIGateway", "OCIEvents", - "OCILoadBalancer", "OCIArtifacts", "OCIContainerInstances", + "OCILoadBalancer", "OCIArtifacts", "OCIContainerInstances", "OCIDNS", "OCIDatabase", + "OCIFileStorage", } for _, e := range entries { diff --git a/internal/app/servicecatalog_wiring.go b/internal/app/servicecatalog_wiring.go index 0de2513b58..79f31306a2 100644 --- a/internal/app/servicecatalog_wiring.go +++ b/internal/app/servicecatalog_wiring.go @@ -167,7 +167,7 @@ func (s *serviceCatalogStacks) CreateStack( return servicecatalogbackend.StackInfo{}, err } - return stackInfoOf(st), nil + return settledStackInfo(ctx, bk, st) } func (s *serviceCatalogStacks) UpdateStack( @@ -184,7 +184,7 @@ func (s *serviceCatalogStacks) UpdateStack( return servicecatalogbackend.StackInfo{}, err } - return stackInfoOf(st), nil + return settledStackInfo(ctx, bk, st) } func (s *serviceCatalogStacks) DeleteStack(ctx context.Context, nameOrID string) error { @@ -193,7 +193,28 @@ func (s *serviceCatalogStacks) DeleteStack(ctx context.Context, nameOrID string) return err } - return bk.DeleteStack(ctx, nameOrID) + if err = bk.DeleteStack(ctx, nameOrID); err != nil { + return err + } + + _, err = bk.WaitForStack(ctx, nameOrID) + if errors.Is(err, cfnbackend.ErrStackNotFound) { + return nil + } + + return err +} + +// settledStackInfo waits out the stack's async provisioning, as callers here need its final status. +func settledStackInfo( + ctx context.Context, bk cfnbackend.StorageBackend, st *cfnbackend.Stack, +) (servicecatalogbackend.StackInfo, error) { + settled, err := bk.WaitForStack(ctx, st.StackID) + if err != nil { + return servicecatalogbackend.StackInfo{}, err + } + + return stackInfoOf(settled), nil } func (s *serviceCatalogStacks) DescribeStack( @@ -279,7 +300,13 @@ func (s *serviceCatalogStacks) ExecuteChangeSet(ctx context.Context, stackName, return err } - return bk.ExecuteChangeSet(ctx, stackName, changeSetName, false, false) + if err = bk.ExecuteChangeSet(ctx, stackName, changeSetName, false, false); err != nil { + return err + } + + _, err = bk.WaitForStack(ctx, stackName) + + return err } func (s *serviceCatalogStacks) DeleteChangeSet(ctx context.Context, stackName, changeSetName string) error { diff --git a/internal/app/transcribe_output.go b/internal/app/transcribe_output.go new file mode 100644 index 0000000000..802e596de8 --- /dev/null +++ b/internal/app/transcribe_output.go @@ -0,0 +1,19 @@ +package app + +import ( + "net/http" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + transcribebackend "github.com/blackbirdworks/gopherstack/services/transcribe" +) + +// registerTranscribeEndpoints serves the fetch URLs of service-managed Transcribe output. +func registerTranscribeEndpoints(e *echo.Echo, services []service.Registerable) { + for _, svc := range services { + if h, ok := svc.(*transcribebackend.Handler); ok { + e.Add(http.MethodGet, "/_gopherstack/transcribe/output/*", h.ServeManagedOutput) + } + } +} diff --git a/internal/app/transcribe_output_test.go b/internal/app/transcribe_output_test.go new file mode 100644 index 0000000000..d1d343e486 --- /dev/null +++ b/internal/app/transcribe_output_test.go @@ -0,0 +1,48 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + transcribebackend "github.com/blackbirdworks/gopherstack/services/transcribe" +) + +func TestRegisterTranscribeEndpoints(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantCode int + }{ + { + name: "unknown job", + path: "/_gopherstack/transcribe/output/us-east-1/transcription/nope/tok/asrOutput.json", + wantCode: http.StatusForbidden, + }, + {name: "short path", path: "/_gopherstack/transcribe/output/us-east-1/x", wantCode: http.StatusNotFound}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := transcribebackend.NewHandler(transcribebackend.NewInMemoryBackend()) + h.EnableRegions() + + e := echo.New() + registerTranscribeEndpoints(e, []service.Registerable{h}) + + rec := httptest.NewRecorder() + e.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, tt.path, nil)) + require.Equal(t, tt.wantCode, rec.Code) + assert.Empty(t, rec.Body.String()) + }) + } +} diff --git a/internal/cliconfig/lifecycle.go b/internal/cliconfig/lifecycle.go index 830bfde521..c8fb5df5b2 100644 --- a/internal/cliconfig/lifecycle.go +++ b/internal/cliconfig/lifecycle.go @@ -63,6 +63,7 @@ type LifecycleSettings struct { AppStream time.Duration `name:"appstream" env:"APPSTREAM_LIFECYCLE_DELAY" default:"0s" help:"AppStream fleet and image builder STARTING/STOPPING/PENDING window."` //nolint:lll // config struct tags are intentionally verbose FIS time.Duration `name:"fis" env:"FIS_LIFECYCLE_DELAY" default:"0s" help:"FIS experiment state step delay (built-in 10ms)."` //nolint:lll // config struct tags are intentionally verbose DSQL time.Duration `name:"dsql" env:"DSQL_LIFECYCLE_DELAY" default:"0s" help:"Aurora DSQL cluster and stream transitional window (built-in 750ms/500ms)."` //nolint:lll // config struct tags are intentionally verbose + LambdaWeb time.Duration `name:"lambdaweb" env:"LAMBDAWEB_LIFECYCLE_DELAY" default:"0s" help:"Lambda Web Functions function, revision and endpoint transitional window (built-in 500ms-1s)."` //nolint:lll // config struct tags are intentionally verbose GlobalAccelerator time.Duration `name:"globalaccelerator" env:"GLOBALACCELERATOR_LIFECYCLE_DELAY" default:"0s" help:"Global Accelerator transitional window (built-in 500ms)."` //nolint:lll // config struct tags are intentionally verbose CodeGuruReviewer time.Duration `name:"codegurureviewer" env:"CODEGURUREVIEWER_LIFECYCLE_DELAY" default:"0s" help:"CodeGuru Reviewer repository association and code review transitional window (built-in 500ms)."` //nolint:lll // config struct tags are intentionally verbose Connect time.Duration `name:"connect" env:"CONNECT_LIFECYCLE_DELAY" default:"0s" help:"Amazon Connect instance CREATION_IN_PROGRESS window (built-in 1s)."` //nolint:lll // config struct tags are intentionally verbose diff --git a/internal/routegates/gates.go b/internal/routegates/gates.go index 36eee80d3e..5b9a60efb1 100644 --- a/internal/routegates/gates.go +++ b/internal/routegates/gates.go @@ -143,7 +143,8 @@ func AltGates() map[string]service.AltGate { "/.well-known/jwks.json", "/.well-known/openid-configuration", "/saml2/metadata", }, }, - "Lambda": {Target: []string{"AWSLambda"}, Path: []string{"/20"}, Host: []string{".lambda-url."}}, + "Lambda": {Target: []string{"AWSLambda"}, Path: []string{"/20"}, Host: []string{".lambda-url."}}, + "LambdaWeb": {Path: []string{"/2025-03-07/"}, Host: []string{".lambda-web."}}, "EventBridge": { Target: []string{"AmazonEventBridge.", "AWSEvents.", "AWSSchemas."}, Path: []string{"/v1"}, }, diff --git a/pkgs/devtls/ca.go b/pkgs/devtls/ca.go index 8488a7a304..6979d6c69c 100644 --- a/pkgs/devtls/ca.go +++ b/pkgs/devtls/ca.go @@ -65,6 +65,9 @@ func NewCA() (*CA, error) { // CertPEM returns the CA certificate in PEM form, for clients to trust. func (c *CA) CertPEM() []byte { return append([]byte(nil), c.certPEM...) } +// Certificate returns the parsed CA certificate. +func (c *CA) Certificate() *x509.Certificate { return c.cert } + // Pool returns a cert pool containing only this CA. func (c *CA) Pool() *x509.CertPool { p := x509.NewCertPool() @@ -76,6 +79,15 @@ func (c *CA) Pool() *x509.CertPool { // Issue signs a server leaf certificate covering the given DNS names and IPs // (each entry is tried as an IP first). func (c *CA) Issue(hosts ...string) (*tls.Certificate, error) { + return c.issue(x509.ExtKeyUsageServerAuth, orgName, hosts) +} + +// IssueClient signs a client-authentication leaf certificate with the given common name. +func (c *CA) IssueClient(commonName string) (*tls.Certificate, error) { + return c.issue(x509.ExtKeyUsageClientAuth, commonName, nil) +} + +func (c *CA) issue(usage x509.ExtKeyUsage, commonName string, hosts []string) (*tls.Certificate, error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, fmt.Errorf("devtls: generate leaf key: %w", err) @@ -90,11 +102,11 @@ func (c *CA) Issue(hosts ...string) (*tls.Certificate, error) { tmpl := &x509.Certificate{ SerialNumber: serial, - Subject: pkix.Name{CommonName: orgName, Organization: []string{orgName}}, + Subject: pkix.Name{CommonName: commonName, Organization: []string{orgName}}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(DefaultValidity), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{usage}, DNSNames: dnsNames, IPAddresses: ips, } diff --git a/pkgs/devtls/cert_test.go b/pkgs/devtls/cert_test.go index 024c6593aa..b77f32232e 100644 --- a/pkgs/devtls/cert_test.go +++ b/pkgs/devtls/cert_test.go @@ -127,3 +127,41 @@ func TestLoadOrCreateCA_Persists(t *testing.T) { _, err = parsed.Verify(x509.VerifyOptions{Roots: first.Pool(), DNSName: "x.amazonaws.com"}) require.NoError(t, err) } + +func TestCA_IssueClient(t *testing.T) { + t.Parallel() + + ca, err := devtls.NewCA() + require.NoError(t, err) + + tests := []struct { + name string + usage x509.ExtKeyUsage + want bool + }{ + {name: "client-auth", usage: x509.ExtKeyUsageClientAuth, want: true}, + {name: "server-auth", usage: x509.ExtKeyUsageServerAuth, want: false}, + } + + leaf, err := ca.IssueClient("wallet-user") + require.NoError(t, err) + + parsed, err := x509.ParseCertificate(leaf.Certificate[0]) + require.NoError(t, err) + assert.Equal(t, "wallet-user", parsed.Subject.CommonName) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, verr := parsed.Verify(x509.VerifyOptions{Roots: ca.Pool(), KeyUsages: []x509.ExtKeyUsage{tc.usage}}) + if tc.want { + require.NoError(t, verr) + } else { + require.Error(t, verr) + } + }) + } + + assert.Equal(t, "gopherstack local CA", ca.Certificate().Subject.CommonName) +} diff --git a/pkgs/dmsengine/dmstypes.go b/pkgs/dmsengine/dmstypes.go new file mode 100644 index 0000000000..9ad0b92b61 --- /dev/null +++ b/pkgs/dmsengine/dmstypes.go @@ -0,0 +1,161 @@ +package dmsengine + +import ( + "fmt" + "strings" +) + +// DMSType is a documented AWS DMS data type with its length, precision and scale. +type DMSType struct { + Name string + Length int + Precision int + Scale int +} + +// dmsTypeKinds maps the DMS data types the transformation rules and S3 sources accept to a value class. +func dmsTypeKinds() map[string]Kind { + return map[string]Kind{ + tBytes: KindBytes, tBlob: KindBytes, + tDate: KindTime, tTime: KindTime, "datetime": KindTime, + tInt1: KindInt, tInt2: KindInt, tInt4: KindInt, tInt8: KindInt, + "uint1": KindInt, "uint2": KindInt, "uint4": KindInt, "uint8": KindInt, + "numeric": KindDecimal, tReal4: KindFloat, tReal8: KindFloat, + "string": KindString, "wstring": KindString, tClob: KindString, tNclob: KindString, + "boolean": KindBool, + } +} + +// ParseDMSType validates a DMS data type name. +func ParseDMSType(name string) (DMSType, error) { + n := strings.ToLower(strings.TrimSpace(name)) + if _, ok := dmsTypeKinds()[n]; !ok { + return DMSType{}, fmt.Errorf("%w: unsupported data-type %q", ErrMapping, name) + } + + return DMSType{Name: n}, nil +} + +// Kind is the value class of the type. +func (d DMSType) Kind() Kind { return dmsTypeKinds()[d.Name] } + +// Column derives a nullable column of the type. +func (d DMSType) Column(name string) Column { + return Column{Name: name, Native: d.Name, Kind: d.Kind(), Nullable: true, DMS: d} +} + +// dmsTypeOf maps an engine-native type to a DMS type name; empty when unknown. +func dmsTypeOf(native string) string { + p := parseNative(native) + m := map[string]string{ + tTinyint: tInt1, tSmallint: tInt2, tInt2: tInt2, tMediumint: tInt4, tInt: tInt4, tInteger: tInt4, + tInt4: tInt4, tSerial: tInt4, tBigint: tInt8, tInt8: tInt8, tBigserial: tInt8, + tDecimal: tNumeric, tNumeric: tNumeric, tFloat: tReal4, tReal: tReal4, tFloat4: tReal4, + tDouble: tReal8, tDoublePrec: tReal8, "float8": tReal8, tBoolean: tBoolean, tBool: tBoolean, + tDate: tDate, tTime: tTime, tDatetime: tDatetime, tTimestamp: tDatetime, tTimestampN: tDatetime, + tTimestampZ: tDatetime, tVarchar: tString, tChar: tString, tText: tString, "character varying": tString, + "character": tString, "bpchar": tString, tLongtext: tClob, "mediumtext": tClob, "tinytext": tString, + tBlob: tBlob, tLongblob: tBlob, tBytea: tBlob, tBinary: tBytes, tVarbinary: tBytes, + } + + return m[p.base] +} + +const ( + maxDatetimeScale = 6 + maxMySQLVarchar = 16383 + maxMySQLVarbin = 65535 +) + +// fixedSQLTypes are the DMS types whose SQL form takes no length, precision or scale: {PostgreSQL, MySQL}. +func fixedSQLTypes() map[string][2]string { + return map[string][2]string{ + tInt1: { + tSmallint, + "tinyint", + }, tInt2: {tSmallint, tSmallint}, tInt4: {tInteger, tInt}, tInt8: {tBigint, tBigint}, + "uint1": {tSmallint, "tinyint unsigned"}, "uint2": {tInteger, "smallint unsigned"}, + "uint4": {tBigint, "int unsigned"}, "uint8": {"numeric(20)", "bigint unsigned"}, + tReal4: {tReal, tFloat}, tReal8: {tDoublePrec, tDouble}, tBoolean: {tBoolean, tTinyint1}, + tDate: {tDate, tDate}, tTime: {tTime, tTime}, + } +} + +// SQLType renders the DMS type for a SQL target of family fam. +func (d DMSType) SQLType(fam Family) string { + pg := fam == FamilyPostgres + + if v, ok := fixedSQLTypes()[d.Name]; ok { + return pick(pg, v[0], v[1]) + } + + switch d.Name { + case tNumeric: + return d.numeric() + case tDatetime: + return d.datetime(pg) + case tBytes, tBlob: + return d.binary(pg) + default: + return d.text(pg) + } +} + +func pick(first bool, a, b string) string { + if first { + return a + } + + return b +} + +func (d DMSType) numeric() string { + switch { + case d.Precision > 0 && d.Scale > 0: + return fmt.Sprintf("numeric(%d,%d)", d.Precision, d.Scale) + case d.Precision > 0: + return fmt.Sprintf("numeric(%d)", d.Precision) + default: + return tNumeric + } +} + +func (d DMSType) datetime(pg bool) string { + scale := d.Scale + if scale == 0 { + scale = d.Precision + } + + base := pick(pg, tTimestamp, tDatetime) + if scale > 0 { + return fmt.Sprintf("%s(%d)", base, min(scale, maxDatetimeScale)) + } + + return base +} + +func (d DMSType) binary(pg bool) string { + switch { + case pg: + return tBytea + case d.Name == tBytes && d.Length > 0 && d.Length <= maxMySQLVarbin: + return fmt.Sprintf("varbinary(%d)", d.Length) + default: + return tLongblob + } +} + +func (d DMSType) text(pg bool) string { + long := d.Name == tClob || d.Name == tNclob || d.Length <= 0 + + switch { + case pg && long: + return tText + case pg: + return fmt.Sprintf("varchar(%d)", d.Length) + case long || d.Length > maxMySQLVarchar: + return tLongtext + default: + return fmt.Sprintf("varchar(%d)", d.Length) + } +} diff --git a/pkgs/dmsengine/endpoint.go b/pkgs/dmsengine/endpoint.go index d4591ed036..596f53eaa6 100644 --- a/pkgs/dmsengine/endpoint.go +++ b/pkgs/dmsengine/endpoint.go @@ -32,7 +32,13 @@ type EndpointConfig struct { // SlotName names the PostgreSQL logical replication slot for source endpoints. SlotName string // S3Settings, KinesisSettings and DynamoDBSettings carry the raw settings JSON objects. - S3Settings string + S3Settings string + // ExternalTableDefinition is the S3 source's table-definition JSON (the endpoint member or S3Settings' + // ExternalTableDefinition). + ExternalTableDefinition string + // SecretID and SecretRoleARN are SecretsManagerSecretId and SecretsManagerAccessRoleArn from the engine settings. + SecretID string + SecretRoleARN string KinesisSettings string DynamoDBSettings string Port int @@ -51,6 +57,9 @@ func (c EndpointConfig) addr() string { // ObjectStore writes objects into the emulator's S3. type ObjectStore interface { PutObject(ctx context.Context, bucket, key string, body []byte) error + // ListObjects returns the keys under prefix in lexicographic order. + ListObjects(ctx context.Context, bucket, prefix string) ([]string, error) + GetObject(ctx context.Context, bucket, key string) ([]byte, error) CheckBucket(ctx context.Context, bucket string) error } @@ -100,7 +109,7 @@ type Opener interface { func SupportedSource(engine string) bool { f := FamilyOf(engine) - return f == FamilyMySQL || f == FamilyPostgres + return f == FamilyMySQL || f == FamilyPostgres || f == FamilyS3 } // SupportedTarget reports whether the engine can write to the endpoint engine. @@ -119,6 +128,8 @@ func (o *defaultOpener) OpenSource(ctx context.Context, ep EndpointConfig) (Sour return openMySQLSource(ctx, ep) case FamilyPostgres: return openPGSource(ctx, ep) + case FamilyS3: + return newS3Source(ep, o.stores.Objects) default: return nil, fmt.Errorf("%w: %s as a source", ErrUnsupportedEngine, ep.Engine) } diff --git a/pkgs/dmsengine/expr.go b/pkgs/dmsengine/expr.go new file mode 100644 index 0000000000..9536e66db6 --- /dev/null +++ b/pkgs/dmsengine/expr.go @@ -0,0 +1,1351 @@ +package dmsengine + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "math/rand/v2" + "strconv" + "strings" + "time" + "unicode" +) + +const ( + opWordInsert = "INSERT" + arityTwo = 2 + arityThree = 3 + blobRank = 2 + decimalBase = 10 + nanosPerMilli = 1_000_000 + daysInWeek = 7 +) + +// ErrExpression reports an add-column expression the engine cannot evaluate. +var ErrExpression = fmt.Errorf("%w: unsupported expression", ErrMapping) + +// EvalCtx is what an expression sees: the source row and the change header values. +type EvalCtx struct { + Time time.Time + Now time.Time + Op Op + Position string + Row Row + Full bool +} + +// Expr is a compiled SQLite-style expression from a transformation rule. +type Expr struct{ eval node } + +type node func(*EvalCtx) any + +// Eval evaluates the expression: nil, int64, float64, string or []byte. +func (e *Expr) Eval(c *EvalCtx) any { return e.eval(c) } + +type tokKind int + +const ( + tokEOF tokKind = iota + tokNum + tokStr + tokIdent + tokRef + tokOp +) + +type token struct { + text string + kind tokKind +} + +func lex(src string) ([]token, error) { + var out []token + + rs := []rune(src) + + for i := 0; i < len(rs); { + r := rs[i] + + switch { + case unicode.IsSpace(r): + i++ + case r == '\'': + s, n, err := lexString(rs[i:]) + if err != nil { + return nil, err + } + + out, i = append(out, token{s, tokStr}), i+n + case unicode.IsDigit(r) || (r == '.' && i+1 < len(rs) && unicode.IsDigit(rs[i+1])): + n := scanWhile(rs[i:], func(c rune) bool { return unicode.IsDigit(c) || c == '.' }) + out, i = append(out, token{string(rs[i : i+n]), tokNum}), i+n + case r == '$': + n := scanWhile(rs[i+1:], isIdentRune) + if n == 0 { + return nil, fmt.Errorf("%w: empty column reference", ErrExpression) + } + + out, i = append(out, token{string(rs[i+1 : i+1+n]), tokRef}), i+1+n + case isIdentRune(r): + n := scanWhile(rs[i:], isIdentRune) + out, i = append(out, token{string(rs[i : i+n]), tokIdent}), i+n + default: + op, n := lexOp(rs[i:]) + if n == 0 { + return nil, fmt.Errorf("%w: unexpected character %q", ErrExpression, r) + } + + out, i = append(out, token{op, tokOp}), i+n + } + } + + return append(out, token{kind: tokEOF}), nil +} + +func isIdentRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' } + +func scanWhile(rs []rune, f func(rune) bool) int { + n := 0 + for n < len(rs) && f(rs[n]) { + n++ + } + + return n +} + +func lexString(rs []rune) (string, int, error) { + var sb strings.Builder + + for i := 1; i < len(rs); i++ { + if rs[i] != '\'' { + sb.WriteRune(rs[i]) + + continue + } + + if i+1 < len(rs) && rs[i+1] == '\'' { + sb.WriteRune('\'') + + i++ + + continue + } + + return sb.String(), i + 1, nil + } + + return "", 0, fmt.Errorf("%w: unterminated string literal", ErrExpression) +} + +func lexOp(rs []rune) (string, int) { + for _, op := range []string{"||", "<>", "!=", "<=", ">=", "=="} { + if strings.HasPrefix(string(rs), op) { + return op, len(op) + } + } + + if strings.ContainsRune("+-*/%=<>(),", rs[0]) { + return string(rs[0]), 1 + } + + return "", 0 +} + +type parser struct { + table Table + toks []token + pos int +} + +// CompileExpr compiles an expression against the source table whose columns $name refers to. +func CompileExpr(src string, t Table) (*Expr, error) { + toks, err := lex(src) + if err != nil { + return nil, err + } + + p := &parser{toks: toks, table: t} + + n, err := p.parseOr() + if err != nil { + return nil, err + } + + if p.peek().kind != tokEOF { + return nil, fmt.Errorf("%w: unexpected %q", ErrExpression, p.peek().text) + } + + return &Expr{eval: n}, nil +} + +func (p *parser) peek() token { return p.toks[p.pos] } + +func (p *parser) next() token { + t := p.toks[p.pos] + if t.kind != tokEOF { + p.pos++ + } + + return t +} + +func (p *parser) isKeyword(kw string) bool { + t := p.peek() + + return t.kind == tokIdent && strings.EqualFold(t.text, kw) +} + +func (p *parser) acceptKeyword(kw string) bool { + if p.isKeyword(kw) { + p.pos++ + + return true + } + + return false +} + +func (p *parser) acceptOp(ops ...string) (string, bool) { + t := p.peek() + if t.kind != tokOp { + return "", false + } + + for _, o := range ops { + if t.text == o { + p.pos++ + + return o, true + } + } + + return "", false +} + +func (p *parser) expectOp(op string) error { + if _, ok := p.acceptOp(op); !ok { + return fmt.Errorf("%w: expected %q, found %q", ErrExpression, op, p.peek().text) + } + + return nil +} + +func (p *parser) parseOr() (node, error) { + l, err := p.parseAnd() + if err != nil { + return nil, err + } + + for p.acceptKeyword("or") { + r, rerr := p.parseAnd() + if rerr != nil { + return nil, rerr + } + + l = logic(l, r, true) + } + + return l, nil +} + +func (p *parser) parseAnd() (node, error) { + l, err := p.parseNot() + if err != nil { + return nil, err + } + + for p.acceptKeyword("and") { + r, rerr := p.parseNot() + if rerr != nil { + return nil, rerr + } + + l = logic(l, r, false) + } + + return l, nil +} + +func (p *parser) parseNot() (node, error) { + if !p.acceptKeyword("not") { + return p.parseCompare() + } + + n, err := p.parseNot() + if err != nil { + return nil, err + } + + return func(c *EvalCtx) any { + v := n(c) + if v == nil { + return nil + } + + return boolInt(!truthy(v)) + }, nil +} + +func (p *parser) parseCompare() (node, error) { + l, err := p.parseAdd() + if err != nil { + return nil, err + } + + for { + if p.acceptKeyword("is") { + l, err = p.parseIs(l) + if err != nil { + return nil, err + } + + continue + } + + op, ok := p.acceptOp("=", "==", "<>", "!=", "<", "<=", ">", ">=") + if !ok { + return l, nil + } + + r, rerr := p.parseAdd() + if rerr != nil { + return nil, rerr + } + + l = compareNode(op, l, r) + } +} + +func (p *parser) parseIs(l node) (node, error) { + neg := p.acceptKeyword("not") + + if p.acceptKeyword("null") { + return func(c *EvalCtx) any { return boolInt((l(c) == nil) != neg) }, nil + } + + r, err := p.parseAdd() + if err != nil { + return nil, err + } + + return func(c *EvalCtx) any { + a, b := l(c), r(c) + same := (a == nil && b == nil) || (a != nil && b != nil && compareValues(a, b) == 0) + + return boolInt(same != neg) + }, nil +} + +func (p *parser) parseAdd() (node, error) { + l, err := p.parseMul() + if err != nil { + return nil, err + } + + for { + op, ok := p.acceptOp("+", "-") + if !ok { + return l, nil + } + + r, rerr := p.parseMul() + if rerr != nil { + return nil, rerr + } + + l = arith(op, l, r) + } +} + +func (p *parser) parseMul() (node, error) { + l, err := p.parseConcat() + if err != nil { + return nil, err + } + + for { + op, ok := p.acceptOp("*", "/", "%") + if !ok { + return l, nil + } + + r, rerr := p.parseConcat() + if rerr != nil { + return nil, rerr + } + + l = arith(op, l, r) + } +} + +func (p *parser) parseConcat() (node, error) { + l, err := p.parseUnary() + if err != nil { + return nil, err + } + + for { + if _, ok := p.acceptOp("||"); !ok { + return l, nil + } + + r, rerr := p.parseUnary() + if rerr != nil { + return nil, rerr + } + + a, b := l, r + l = func(c *EvalCtx) any { + x, y := a(c), b(c) + if x == nil || y == nil { + return nil + } + + return text(x) + text(y) + } + } +} + +func (p *parser) parseUnary() (node, error) { + if op, ok := p.acceptOp("-", "+"); ok { + n, err := p.parseUnary() + if err != nil { + return nil, err + } + + if op == "+" { + return n, nil + } + + return func(c *EvalCtx) any { return arithValues("*", int64(-1), n(c)) }, nil + } + + return p.parsePrimary() +} + +func (p *parser) parsePrimary() (node, error) { + t := p.next() + + switch t.kind { + case tokNum: + return numLiteral(t.text) + case tokStr: + return func(*EvalCtx) any { return t.text }, nil + case tokRef: + return p.reference(t.text) + case tokIdent: + return p.identifier(t.text) + case tokOp: + if t.text == "(" { + n, err := p.parseOr() + if err != nil { + return nil, err + } + + return n, p.expectOp(")") + } + case tokEOF: + return nil, fmt.Errorf("%w: unexpected end of expression", ErrExpression) + } + + return nil, fmt.Errorf("%w: unexpected %q", ErrExpression, t.text) +} + +func numLiteral(s string) (node, error) { + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return func(*EvalCtx) any { return i }, nil + } + + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, fmt.Errorf("%w: bad number %q", ErrExpression, s) + } + + return func(*EvalCtx) any { return f }, nil +} + +func (p *parser) identifier(name string) (node, error) { + switch strings.ToLower(name) { + case "null": + return func(*EvalCtx) any { return nil }, nil + case "case": + return p.parseCase() + } + + if _, ok := p.acceptOp("("); !ok { + return nil, fmt.Errorf("%w: unexpected identifier %q", ErrExpression, name) + } + + return p.call(strings.ToLower(name)) +} + +func (p *parser) call(name string) (node, error) { + var args []node + + if _, ok := p.acceptOp(")"); !ok { + for { + a, err := p.parseOr() + if err != nil { + return nil, err + } + + args = append(args, a) + + if _, more := p.acceptOp(","); !more { + break + } + } + + if err := p.expectOp(")"); err != nil { + return nil, err + } + } + + spec, ok := exprFuncs()[name] + if !ok { + return nil, fmt.Errorf("%w: function %s() is not supported", ErrExpression, name) + } + + if len(args) < spec.min || (spec.max >= 0 && len(args) > spec.max) { + return nil, fmt.Errorf("%w: wrong number of arguments to %s()", ErrExpression, name) + } + + return func(c *EvalCtx) any { + vals := make([]any, len(args)) + for i, a := range args { + vals[i] = a(c) + } + + return spec.fn(c, vals) + }, nil +} + +type caseArm struct{ when, then node } + +func (p *parser) parseCase() (node, error) { + var subject node + + if !p.isKeyword("when") { + s, err := p.parseOr() + if err != nil { + return nil, err + } + + subject = s + } + + var arms []caseArm + + for p.acceptKeyword("when") { + arm, err := p.parseArm() + if err != nil { + return nil, err + } + + arms = append(arms, arm) + } + + var els node + + if p.acceptKeyword("else") { + e, err := p.parseOr() + if err != nil { + return nil, err + } + + els = e + } + + if len(arms) == 0 || !p.acceptKeyword("end") { + return nil, fmt.Errorf("%w: malformed CASE expression", ErrExpression) + } + + return func(c *EvalCtx) any { return evalCase(c, subject, arms, els) }, nil +} + +func (p *parser) parseArm() (caseArm, error) { + w, err := p.parseOr() + if err != nil { + return caseArm{}, err + } + + if !p.acceptKeyword("then") { + return caseArm{}, fmt.Errorf("%w: CASE WHEN needs THEN", ErrExpression) + } + + th, err := p.parseOr() + + return caseArm{when: w, then: th}, err +} + +func evalCase(c *EvalCtx, subject node, arms []caseArm, els node) any { + var base any + if subject != nil { + base = subject(c) + } + + for _, a := range arms { + w := a.when(c) + + matched := w != nil && truthy(w) + if subject != nil { + matched = base != nil && w != nil && compareValues(base, w) == 0 + } + + if matched { + return a.then(c) + } + } + + if els != nil { + return els(c) + } + + return nil +} + +func (p *parser) reference(name string) (node, error) { + if strings.HasPrefix(name, "AR_") { + return p.header(name) + } + + for i, c := range p.table.Columns { + if c.Name == name { + kind := c.Kind + + return func(ctx *EvalCtx) any { + if i >= len(ctx.Row) { + return nil + } + + return fromText(ctx.Row[i], kind) + }, nil + } + } + + return nil, fmt.Errorf("%w: column $%s does not exist in %s", ErrExpression, name, p.table.Key()) +} + +const headerTimeLayout = "2006-01-02 15:04:05.000000" + +func (p *parser) header(name string) (node, error) { + switch name { + case "AR_H_STREAM_POSITION": + return func(c *EvalCtx) any { + if c.Full { + return "" + } + + return c.Position + }, nil + case "AR_H_TIMESTAMP", "AR_H_COMMIT_TIMESTAMP": + return func(c *EvalCtx) any { + if c.Full || c.Time.IsZero() { + return c.Now.UTC().Format(headerTimeLayout) + } + + return c.Time.UTC().Format(headerTimeLayout) + }, nil + case "AR_H_OPERATION": + return func(c *EvalCtx) any { + if c.Full { + return opWordInsert + } + + return strings.ToUpper(string(c.Op)) + }, nil + case "AR_H_USER": + return func(*EvalCtx) any { return "" }, nil + case "AR_M_SOURCE_SCHEMA": + s := p.table.Schema + + return func(*EvalCtx) any { return s }, nil + case "AR_M_SOURCE_TABLE_NAME": + s := p.table.Name + + return func(*EvalCtx) any { return s }, nil + } + + return nil, fmt.Errorf("%w: $%s is not supported", ErrExpression, name) +} + +func fromText(v any, k Kind) any { + switch x := v.(type) { + case nil: + return nil + case []byte: + return x + case string: + switch k { + case KindInt: + if i, err := strconv.ParseInt(x, 10, 64); err == nil { + return i + } + case KindDecimal, KindFloat: + if f, err := strconv.ParseFloat(x, 64); err == nil { + return f + } + case KindString, KindBool, KindTime, KindBytes, KindJSON: + } + + return x + } + + return fmt.Sprint(v) +} + +func boolInt(b bool) any { + if b { + return int64(1) + } + + return int64(0) +} + +func truthy(v any) bool { + switch x := toNumber(v).(type) { + case int64: + return x != 0 + case float64: + return x != 0 + } + + return false +} + +// toNumber follows SQLite's numeric prefix rule: text that is not a number reads as 0. +func toNumber(v any) any { + switch x := v.(type) { + case int64, float64: + return x + case []byte: + return toNumber(string(x)) + case string: + s := strings.TrimSpace(x) + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return i + } + + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f + } + + return numericPrefix(s) + } + + return int64(0) +} + +func numericPrefix(s string) any { + for end := len(s); end > 0; end-- { + if f, err := strconv.ParseFloat(s[:end], 64); err == nil { + if i, ierr := strconv.ParseInt(s[:end], 10, 64); ierr == nil { + return i + } + + return f + } + } + + return int64(0) +} + +func asFloat(v any) float64 { + switch x := toNumber(v).(type) { + case int64: + return float64(x) + case float64: + return x + } + + return 0 +} + +func logic(l, r node, or bool) node { + return func(c *EvalCtx) any { + a, b := l(c), r(c) + at, bt := a != nil && truthy(a), b != nil && truthy(b) + + switch { + case or && (at || bt): + return int64(1) + case !or && ((a != nil && !at) || (b != nil && !bt)): + return int64(0) + case a == nil || b == nil: + return nil + case or: + return int64(0) + } + + return int64(1) + } +} + +// compareValues orders values SQLite-style: numbers before text before blobs. +func compareValues(a, b any) int { + switch x := a.(type) { + case string: + if y, ok := b.(string); ok { + return strings.Compare(x, y) + } + case []byte: + if y, ok := b.([]byte); ok { + return bytes.Compare(x, y) + } + default: + if valueRank(b) == 0 { + return compareNumbers(a, b) + } + } + + return valueRank(a) - valueRank(b) +} + +func compareNumbers(a, b any) int { + ia, aInt := a.(int64) + ib, bInt := b.(int64) + + if aInt && bInt { + return cmpOrdered(ia, ib) + } + + return cmpOrdered(asFloat(a), asFloat(b)) +} + +func cmpOrdered[T int64 | float64](a, b T) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + + return 0 +} + +func valueRank(v any) int { + switch v.(type) { + case string: + return 1 + case []byte: + return blobRank + } + + return 0 +} + +func compareNode(op string, l, r node) node { + return func(c *EvalCtx) any { + a, b := l(c), r(c) + if a == nil || b == nil { + return nil + } + + d := compareValues(a, b) + + switch op { + case "=", "==": + return boolInt(d == 0) + case "<>", "!=": + return boolInt(d != 0) + case "<": + return boolInt(d < 0) + case "<=": + return boolInt(d <= 0) + case ">": + return boolInt(d > 0) + } + + return boolInt(d >= 0) + } +} + +func arith(op string, l, r node) node { + return func(c *EvalCtx) any { return arithValues(op, l(c), r(c)) } +} + +func arithValues(op string, a, b any) any { + if a == nil || b == nil { + return nil + } + + na, nb := toNumber(a), toNumber(b) + ia, aInt := na.(int64) + ib, bInt := nb.(int64) + + if aInt && bInt { + return intArith(op, ia, ib) + } + + fa, fb := asFloat(na), asFloat(nb) + + switch op { + case "+": + return fa + fb + case "-": + return fa - fb + case "*": + return fa * fb + case "/": + if fb == 0 { + return nil + } + + return fa / fb + } + + if fb == 0 { + return nil + } + + return math.Mod(fa, fb) +} + +func intArith(op string, a, b int64) any { + switch op { + case "+": + return a + b + case "-": + return a - b + case "*": + return a * b + case "/": + if b == 0 { + return nil + } + + return a / b + } + + if b == 0 { + return nil + } + + return a % b +} + +type funcSpec struct { + fn func(*EvalCtx, []any) any + min, max int +} + +func exprFuncs() map[string]funcSpec { + return map[string]funcSpec{ + "lower": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { return mapNil(a[0], asciiCase(false)) }}, + "upper": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { return mapNil(a[0], asciiCase(true)) }}, + "ltrim": {min: 1, max: arityTwo, fn: trimFn(true, false)}, + "rtrim": {min: 1, max: arityTwo, fn: trimFn(false, true)}, + "trim": {min: 1, max: arityTwo, fn: trimFn(true, true)}, + "replace": {min: arityThree, max: arityThree, fn: func(_ *EvalCtx, a []any) any { + if a[0] == nil || a[1] == nil || a[2] == nil { + return nil + } + + if text(a[1]) == "" { + return text(a[0]) + } + + return strings.ReplaceAll(text(a[0]), text(a[1]), text(a[2])) + }}, + "substr": {min: arityTwo, max: arityThree, fn: substrFn}, + "hex": {min: 1, max: 1, fn: hexFn}, + "abs": {min: 1, max: 1, fn: absFn}, + "round": {min: 1, max: arityTwo, fn: roundFn}, + "random": {min: 0, max: 0, fn: randomInt}, + "max": {min: arityTwo, max: -1, fn: extremum(1)}, + "min": {min: arityTwo, max: -1, fn: extremum(-1)}, + "coalesce": {min: arityTwo, max: -1, fn: coalesceFn}, + "ifnull": {min: arityTwo, max: arityTwo, fn: coalesceFn}, + "nullif": {min: arityTwo, max: arityTwo, fn: func(_ *EvalCtx, a []any) any { + if a[0] != nil && a[1] != nil && compareValues(a[0], a[1]) == 0 { + return nil + } + + return a[0] + }}, + "zeroblob": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { + return make([]byte, max(0, int(asFloat(a[0])))) + }}, + "randomblob": {min: 1, max: 1, fn: randomBlob}, + "hash_sha256": {min: 1, max: 1, fn: func(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + sum := sha256.Sum256([]byte(text(a[0]))) + + return hex.EncodeToString(sum[:]) + }}, + "operation_indicator": {min: arityThree, max: arityThree, fn: operationIndicator}, + tDate: {min: 0, max: 1, fn: dateFn("2006-01-02")}, + tTime: {min: 0, max: 1, fn: dateFn("15:04:05")}, + "datetime": {min: 0, max: 1, fn: dateFn("2006-01-02 15:04:05")}, + "julianday": {min: 0, max: 1, fn: julianFn}, + "strftime": {min: 1, max: arityTwo, fn: strftimeFn}, + } +} + +func text(v any) string { + switch x := v.(type) { + case nil: + return "" + case string: + return x + case []byte: + return string(x) + case int64: + return strconv.FormatInt(x, 10) + case float64: + return formatFloat(x) + } + + return fmt.Sprint(v) +} + +func formatFloat(f float64) string { + if math.IsInf(f, 0) || math.IsNaN(f) { + return "" + } + + s := strconv.FormatFloat(f, 'g', 15, 64) + if !strings.ContainsAny(s, ".eE") { + s += ".0" + } + + return s +} + +func mapNil(v any, f func(string) string) any { + if v == nil { + return nil + } + + return f(text(v)) +} + +func asciiCase(upper bool) func(string) string { + return func(s string) string { + b := []byte(s) + for i, c := range b { + switch { + case upper && c >= 'a' && c <= 'z': + b[i] = c - 'a' + 'A' + case !upper && c >= 'A' && c <= 'Z': + b[i] = c - 'A' + 'a' + } + } + + return string(b) + } +} + +func trimFn(left, right bool) func(*EvalCtx, []any) any { + return func(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + cut := " " + if len(a) > 1 { + if a[1] == nil { + return nil + } + + cut = text(a[1]) + } + + s := text(a[0]) + if left { + s = strings.TrimLeft(s, cut) + } + + if right { + s = strings.TrimRight(s, cut) + } + + return s + } +} + +func substrFn(_ *EvalCtx, a []any) any { + if a[0] == nil || a[1] == nil || (len(a) == arityThree && a[2] == nil) { + return nil + } + + rs := []rune(text(a[0])) + start := int(asFloat(a[1])) + length := len(rs) + + if len(a) == arityThree { + length = int(asFloat(a[2])) + } + + if start < 0 { + start = len(rs) + start + 1 + } + + from, to := start, start+length + + if length < 0 { + from, to = start+length, start + } + + from, to = max(from, 1), min(to, len(rs)+1) + if from >= to { + return "" + } + + return string(rs[from-1 : to-1]) +} + +func hexFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + return strings.ToUpper(hex.EncodeToString([]byte(text(a[0])))) +} + +func absFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + switch x := toNumber(a[0]).(type) { + case int64: + return max(x, -x) + case float64: + return math.Abs(x) + } + + return float64(0) +} + +func roundFn(_ *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + digits := 0 + if len(a) == arityTwo { + digits = int(asFloat(a[1])) + } + + scale := math.Pow(decimalBase, float64(digits)) + + return math.Round(asFloat(a[0])*scale) / scale +} + +func extremum(dir int) func(*EvalCtx, []any) any { + return func(_ *EvalCtx, a []any) any { + best := a[0] + + for _, v := range a { + if v == nil { + return nil + } + + if compareValues(v, best)*dir > 0 { + best = v + } + } + + return best + } +} + +func coalesceFn(_ *EvalCtx, a []any) any { + for _, v := range a { + if v != nil { + return v + } + } + + return nil +} + +func randomInt(*EvalCtx, []any) any { + return int64(rand.Uint64()) //nolint:gosec // SQL random() is not a security primitive +} + +func randomBlob(_ *EvalCtx, a []any) any { + b := make([]byte, max(1, int(asFloat(a[0])))) + for i := range b { + b[i] = byte(rand.UintN(256)) //nolint:gosec,mnd // SQL randomblob() + } + + return b +} + +func operationIndicator(c *EvalCtx, a []any) any { + if c.Full { + return a[2] + } + + switch c.Op { + case OpDelete: + return a[0] + case OpUpdate: + return a[1] + case OpInsert: + } + + return a[2] +} + +func parseTimeString(c *EvalCtx, v any) (time.Time, bool) { + s := strings.TrimSpace(text(v)) + if strings.EqualFold(s, "now") { + return c.Now.UTC(), true + } + + for _, layout := range []string{ + "2006-01-02 15:04:05.999999999", "2006-01-02T15:04:05.999999999", "2006-01-02 15:04", "2006-01-02", "15:04:05", + } { + if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { + return t, true + } + } + + return time.Time{}, false +} + +func timeArg(c *EvalCtx, a []any) (time.Time, bool) { + if len(a) == 0 { + return c.Now.UTC(), true + } + + if a[0] == nil { + return time.Time{}, false + } + + return parseTimeString(c, a[0]) +} + +func dateFn(layout string) func(*EvalCtx, []any) any { + return func(c *EvalCtx, a []any) any { + t, ok := timeArg(c, a) + if !ok { + return nil + } + + return t.Format(layout) + } +} + +const ( + secondsPerDay = 86400.0 + unixEpochJulis = 2440587.5 +) + +func julianFn(c *EvalCtx, a []any) any { + t, ok := timeArg(c, a) + if !ok { + return nil + } + + return float64(t.UnixNano())/1e9/secondsPerDay + unixEpochJulis +} + +func strftimeFn(c *EvalCtx, a []any) any { + if a[0] == nil { + return nil + } + + t, ok := timeArg(c, a[1:]) + if !ok { + return nil + } + + var sb strings.Builder + + f := []rune(text(a[0])) + for i := 0; i < len(f); i++ { + if f[i] != '%' || i+1 == len(f) { + sb.WriteRune(f[i]) + + continue + } + + i++ + + s, known := strftimeVar(f[i], t) + if !known { + return nil + } + + sb.WriteString(s) + } + + return sb.String() +} + +func strftimeVar(v rune, t time.Time) (string, bool) { + switch v { + case 'd': + return t.Format("02"), true + case 'H': + return t.Format("15"), true + case 'f': + return fmt.Sprintf("%02d.%03d", t.Second(), t.Nanosecond()/nanosPerMilli), true + case 'j': + return fmt.Sprintf("%03d", t.YearDay()), true + case 'J': + return formatFloat(float64(t.UnixNano())/1e9/secondsPerDay + unixEpochJulis), true + case 'm': + return t.Format("01"), true + case 'M': + return t.Format("04"), true + case 's': + return strconv.FormatInt(t.Unix(), 10), true + case 'S': + return t.Format("05"), true + case 'w': + return strconv.Itoa(int(t.Weekday())), true + case 'W': + return fmt.Sprintf("%02d", (t.YearDay()+daysInWeek-1-int(t.Weekday()))/daysInWeek), true + case 'Y': + return t.Format("2006"), true + case '%': + return "%", true + } + + return "", false +} + +// Cast renders an expression result as the text form of a column of DMS type d. +func (d DMSType) Cast(v any) any { + if v == nil { + return nil + } + + switch d.Kind() { + case KindInt: + switch x := toNumber(v).(type) { + case float64: + return strconv.FormatInt(int64(x), 10) + case int64: + return strconv.FormatInt(x, 10) + } + case KindDecimal, KindFloat: + f := asFloat(v) + if i, ok := toNumber(v).(int64); ok { + return strconv.FormatInt(i, 10) + } + + return strconv.FormatFloat(f, 'f', -1, 64) + case KindBool: + return strconv.FormatBool(truthy(v)) + case KindBytes: + if b, ok := v.([]byte); ok { + return b + } + + return []byte(text(v)) + case KindString, KindTime, KindJSON: + } + + return text(v) +} diff --git a/pkgs/dmsengine/filter.go b/pkgs/dmsengine/filter.go new file mode 100644 index 0000000000..9086bcebc7 --- /dev/null +++ b/pkgs/dmsengine/filter.go @@ -0,0 +1,278 @@ +package dmsengine + +import ( + "fmt" + "strconv" + "strings" +) + +// Selection-rule filter operators (DMS User Guide, "Using source filters"). +const ( + opLTE = "lte" + opSTE = "ste" + opGTE = "gte" + opEQ = "eq" + opNotEQ = "noteq" + opBetween = "between" + opNotBetween = "notbetween" + opNull = "null" + opNotNull = "notnull" +) + +type filterCondition struct { + Op string `json:"filter-operator"` + Value flexString `json:"value"` + Start flexString `json:"start-value"` + End flexString `json:"end-value"` +} + +// RuleFilter is one entry of a selection rule's filters array. +type RuleFilter struct { + Type string `json:"filter-type"` + Column string `json:"column-name"` + Conditions []filterCondition `json:"filter-conditions"` +} + +func validateFilter(f RuleFilter) error { + if f.Type != "source" { + return fmt.Errorf("%w: unsupported filter-type %q", ErrMapping, f.Type) + } + + if f.Column == "" || len(f.Conditions) == 0 { + return fmt.Errorf("%w: a filter needs column-name and filter-conditions", ErrMapping) + } + + for _, c := range f.Conditions { + switch c.Op { + case opLTE, opSTE, opGTE, opEQ, opNotEQ, opBetween, opNotBetween, opNull, opNotNull: + default: + return fmt.Errorf("%w: unsupported filter-operator %q", ErrMapping, c.Op) + } + } + + return nil +} + +func (c filterCondition) negative() bool { return c.Op == opNotEQ || c.Op == opNotBetween } + +func (c filterCondition) inclusive() bool { + switch c.Op { + case opLTE, opSTE, opGTE, opEQ, opBetween: + return true + } + + return false +} + +// effective applies the documented NULL notes: with a negative and a null condition in the same filter, NULL rows are +// never replicated, and without an inclusive condition nothing is. +func (f RuleFilter) effective() ([]filterCondition, bool) { + var neg, null, incl bool + + for _, c := range f.Conditions { + neg, null, incl = neg || c.negative(), null || c.Op == opNull, incl || c.inclusive() + } + + if !neg || !null { + return f.Conditions, true + } + + if !incl { + return nil, false + } + + out := make([]filterCondition, 0, len(f.Conditions)) + + for _, c := range f.Conditions { + if c.Op != opNull { + out = append(out, c) + } + } + + return out, true +} + +// TableFilter is the compiled filters of one selection rule bound to a source table. +type TableFilter struct { + table Table + filters []boundFilter +} + +type boundFilter struct { + conds []filterCondition + col Column + idx int + none bool +} + +func newTableFilter(t Table, fs []RuleFilter) (*TableFilter, error) { + if len(fs) == 0 { + return nil, nil //nolint:nilnil // no filters + } + + tf := &TableFilter{table: t} + + for _, f := range fs { + idx := -1 + + for i, c := range t.Columns { + if c.Name == f.Column { + idx = i + } + } + + if idx < 0 { + return nil, fmt.Errorf("%w: filter column %q does not exist in %s", ErrMapping, f.Column, t.Key()) + } + + conds, ok := f.effective() + tf.filters = append(tf.filters, boundFilter{conds: conds, col: t.Columns[idx], idx: idx, none: !ok}) + } + + return tf, nil +} + +// Match reports whether a source row passes every filter. +func (f *TableFilter) Match(r Row) bool { + if f == nil { + return true + } + + for _, bf := range f.filters { + if bf.none || bf.idx >= len(r) || !bf.match(r[bf.idx]) { + return false + } + } + + return true +} + +func (b boundFilter) match(v any) bool { + for _, c := range b.conds { + if b.matchCond(c, v) { + return true + } + } + + return false +} + +func (b boundFilter) matchCond(c filterCondition, v any) bool { + switch c.Op { + case opNull: + return v == nil + case opNotNull: + return v != nil + } + + if v == nil { + return false + } + + cmp := func(s flexString) int { return compareFilterValue(b.col.Kind, v, string(s)) } + + switch c.Op { + case opLTE, opSTE: + return cmp(c.Value) <= 0 + case opGTE: + return cmp(c.Value) >= 0 + case opEQ: + return cmp(c.Value) == 0 + case opNotEQ: + return cmp(c.Value) != 0 + case opBetween: + return cmp(c.Start) >= 0 && cmp(c.End) <= 0 + } + + return cmp(c.Start) < 0 || cmp(c.End) > 0 +} + +// compareFilterValue compares a column value with a rule literal: numerically for numeric columns, otherwise bytewise +// (SQLite BINARY), which orders the ISO date and time forms DMS filters use. +func compareFilterValue(k Kind, v any, lit string) int { + var s string + + switch x := v.(type) { + case string: + s = x + case []byte: + s = string(x) + } + + if k == KindInt || k == KindDecimal || k == KindFloat { + a, aerr := strconv.ParseFloat(strings.TrimSpace(s), 64) + b, berr := strconv.ParseFloat(strings.TrimSpace(lit), 64) + + if aerr == nil && berr == nil { + return cmpOrdered(a, b) + } + } + + return strings.Compare(s, lit) +} + +// SQL renders the filters as a WHERE predicate with positional arguments for the dialect. +func (f *TableFilter) SQL(fam Family, firstArg int) (string, []any) { + if f == nil { + return "", nil + } + + var ( + parts []string + args []any + ) + + ph := func() string { + args = append(args, nil) + + if fam == FamilyPostgres { + return "$" + strconv.Itoa(firstArg+len(args)-1) + } + + return "?" + } + bind := func(s flexString) string { + p := ph() + args[len(args)-1] = string(s) + + return p + } + + for _, bf := range f.filters { + if bf.none { + return "1 = 0", nil + } + + col := QuoteIdent(fam, bf.col.Name) + alts := make([]string, 0, len(bf.conds)) + + for _, c := range bf.conds { + alts = append(alts, condSQL(c, col, bind)) + } + + parts = append(parts, "("+strings.Join(alts, " OR ")+")") + } + + return strings.Join(parts, " AND "), args +} + +func condSQL(c filterCondition, col string, bind func(flexString) string) string { + switch c.Op { + case opNull: + return col + " IS NULL" + case opNotNull: + return col + " IS NOT NULL" + case opLTE, opSTE: + return col + " <= " + bind(c.Value) + case opGTE: + return col + " >= " + bind(c.Value) + case opEQ: + return col + " = " + bind(c.Value) + case opNotEQ: + return col + " <> " + bind(c.Value) + case opBetween: + return col + " BETWEEN " + bind(c.Start) + " AND " + bind(c.End) + } + + return col + " NOT BETWEEN " + bind(c.Start) + " AND " + bind(c.End) +} diff --git a/pkgs/dmsengine/mapping.go b/pkgs/dmsengine/mapping.go index ee83aa71d7..a8fc9cf924 100644 --- a/pkgs/dmsengine/mapping.go +++ b/pkgs/dmsengine/mapping.go @@ -22,9 +22,20 @@ const ( actExclude = "exclude" actRename = "rename" actRemoveColumn = "remove-column" + actIncludeColumn = "include-column" + actAddColumn = "add-column" + actChangeType = "change-data-type" + actBeforeImage = "add-before-image-columns" + actDefinePK = "define-primary-key" targetSchema = "schema" targetTable = "table" targetColumn = "column" + pkOriginUnique = "unique-index" + pkOriginPrimary = "primary-key" + imageFilterPK = "pk-only" + imageFilterNonLOB = "non-lob" + imageFilterAll = "all" + defaultImagePrefix = "BI_" ) type flexString string @@ -47,20 +58,81 @@ func (f *flexString) UnmarshalJSON(b []byte) error { return nil } +type flexInt int + +func (f *flexInt) UnmarshalJSON(b []byte) error { + var s flexString + if err := s.UnmarshalJSON(b); err != nil { + return err + } + + if strings.TrimSpace(string(s)) == "" { + *f = 0 + + return nil + } + + n, err := strconv.Atoi(strings.TrimSpace(string(s))) + if err != nil { + return fmt.Errorf("%w: %q is not an integer", ErrMapping, string(s)) + } + + *f = flexInt(n) + + return nil +} + +type ruleDataType struct { + Type string `json:"type"` + Length flexInt `json:"length"` + Precision flexInt `json:"precision"` + Scale flexInt `json:"scale"` +} + +func (d ruleDataType) parse() (DMSType, error) { + t, err := ParseDMSType(d.Type) + if err != nil { + return t, err + } + + t.Length, t.Precision, t.Scale = int(d.Length), int(d.Precision), int(d.Scale) + + return t, nil +} + +type primaryKeyDef struct { + Name string `json:"name"` + Origin string `json:"origin"` + Columns []string `json:"columns"` +} + +type beforeImageDef struct { + Prefix *string `json:"column-prefix"` + Suffix string `json:"column-suffix"` + Filter string `json:"column-filter"` +} + // Rule is one entry of a table-mappings document. type Rule struct { Locator struct { - Schema string `json:"schema-name"` - Table string `json:"table-name"` - Column string `json:"column-name"` + Schema string `json:"schema-name"` + Table string `json:"table-name"` + Column string `json:"column-name"` + DataType string `json:"data-type"` } `json:"object-locator"` - ID flexString `json:"rule-id"` - Name flexString `json:"rule-name"` - Type string `json:"rule-type"` - Action string `json:"rule-action"` - Target string `json:"rule-target"` - Value string `json:"value"` - Old string `json:"old-value"` + DataType *ruleDataType `json:"data-type"` + PrimaryKey *primaryKeyDef `json:"primary-key-def"` + BeforeImage *beforeImageDef `json:"before-image-def"` + ID flexString `json:"rule-id"` + Name flexString `json:"rule-name"` + LoadOrder flexString `json:"load-order"` + Type string `json:"rule-type"` + Action string `json:"rule-action"` + Target string `json:"rule-target"` + Value string `json:"value"` + Old string `json:"old-value"` + Expression string `json:"expression"` + Filters []RuleFilter `json:"filters"` } // Mappings is a parsed table-mappings document. @@ -101,20 +173,41 @@ func ruleOrder(r Rule) int { return n } +func hasWildcard(s string) bool { return strings.ContainsAny(s, "%_") } + func validateRule(r Rule) error { switch r.Type { case ruleSelection: - switch r.Action { - case actInclude, actExclude, actExplicit: - return nil - } + return validateSelection(r) case ruleTransformation: return validateTransformation(r) default: return fmt.Errorf("%w: unsupported rule-type %q", ErrMapping, r.Type) } +} - return fmt.Errorf("%w: unsupported selection rule-action %q", ErrMapping, r.Action) +func validateSelection(r Rule) error { + switch r.Action { + case actInclude, actExclude, actExplicit: + default: + return fmt.Errorf("%w: unsupported selection rule-action %q", ErrMapping, r.Action) + } + + if r.Action != actInclude { + return nil + } + + if len(r.Filters) > 0 && (hasWildcard(r.Locator.Schema) || hasWildcard(r.Locator.Table)) { + return fmt.Errorf("%w: filters need an exact schema-name and table-name", ErrMapping) + } + + for _, f := range r.Filters { + if err := validateFilter(f); err != nil { + return err + } + } + + return nil } func validateTransformation(r Rule) error { @@ -128,31 +221,130 @@ func validateTransformation(r Rule) error { case actRename, "add-prefix", "remove-prefix", "replace-prefix", "add-suffix", "remove-suffix", "replace-suffix", "convert-lowercase", "convert-uppercase": return nil - case actRemoveColumn: + case actRemoveColumn, actIncludeColumn: if r.Target == targetColumn { return nil } + case actAddColumn, actChangeType, actBeforeImage: + return validateColumnRule(r) + case actDefinePK: + return validatePrimaryKeyRule(r) } return fmt.Errorf("%w: unsupported transformation rule-action %q for %s", ErrMapping, r.Action, r.Target) } -// MatchPattern matches s against a DMS locator pattern where % matches any run of characters. +func validateColumnRule(r Rule) error { + if r.Target != targetColumn { + return fmt.Errorf("%w: %s needs rule-target column", ErrMapping, r.Action) + } + + switch r.Action { + case actAddColumn, actChangeType: + if r.DataType == nil { + return fmt.Errorf("%w: %s needs a data-type", ErrMapping, r.Action) + } + + if _, err := r.DataType.parse(); err != nil { + return err + } + + if r.Action == actAddColumn && r.Value == "" { + return fmt.Errorf("%w: add-column needs a value naming the new column", ErrMapping) + } + default: + return validateBeforeImage(r) + } + + return nil +} + +func validateBeforeImage(r Rule) error { + def := r.imageDef() + + switch def.Filter { + case imageFilterPK, imageFilterNonLOB, imageFilterAll: + default: + return fmt.Errorf("%w: unsupported column-filter %q", ErrMapping, def.Filter) + } + + if *def.Prefix == "" && def.Suffix == "" { + return fmt.Errorf("%w: before-image-def needs a column-prefix or column-suffix", ErrMapping) + } + + return nil +} + +func (r Rule) imageDef() beforeImageDef { + def := beforeImageDef{Filter: imageFilterPK} + prefix := defaultImagePrefix + def.Prefix = &prefix + + if r.BeforeImage != nil { + def.Suffix = r.BeforeImage.Suffix + + if r.BeforeImage.Prefix != nil { + def.Prefix = r.BeforeImage.Prefix + } + + if r.BeforeImage.Filter != "" { + def.Filter = r.BeforeImage.Filter + } + } + + return def +} + +func validatePrimaryKeyRule(r Rule) error { + if r.Target != targetTable { + return fmt.Errorf("%w: define-primary-key needs rule-target table", ErrMapping) + } + + if hasWildcard(r.Locator.Schema) || hasWildcard(r.Locator.Table) { + return fmt.Errorf("%w: define-primary-key takes no wildcards in schema-name or table-name", ErrMapping) + } + + if r.PrimaryKey == nil || len(r.PrimaryKey.Columns) == 0 { + return fmt.Errorf("%w: define-primary-key needs primary-key-def.columns", ErrMapping) + } + + switch r.PrimaryKey.Origin { + case "", pkOriginPrimary, pkOriginUnique: + return nil + } + + return fmt.Errorf("%w: unsupported primary-key-def origin %q", ErrMapping, r.PrimaryKey.Origin) +} + +// MatchPattern matches s against a DMS locator pattern where % matches any run of characters and _ any one +// character (DMS User Guide, "Wildcards in table mapping"). func MatchPattern(pattern, s string) bool { if pattern == "" { pattern = "%" } - if !strings.Contains(pattern, "%") { + if !hasWildcard(pattern) { return pattern == s } - parts := strings.Split(pattern, "%") - for i, p := range parts { - parts[i] = regexp.QuoteMeta(p) + var sb strings.Builder + + sb.WriteString("^") + + for _, r := range pattern { + switch r { + case '%': + sb.WriteString(".*") + case '_': + sb.WriteString(".") + default: + sb.WriteString(regexp.QuoteMeta(string(r))) + } } - return regexp.MustCompile("^" + strings.Join(parts, ".*") + "$").MatchString(s) + sb.WriteString("$") + + return regexp.MustCompile("(?s)" + sb.String()).MatchString(s) } func applyNameAction(action, name, value, old string) string { @@ -184,28 +376,48 @@ func applyNameAction(action, name, value, old string) string { return name } -// Select returns the source tables chosen by the selection rules, in source order. -func (m Mappings) Select(tables []Table) []Table { - chosen := make(map[string]bool, len(tables)) +// SelectedTable is a source table chosen by the selection rules with its source filter. +type SelectedTable struct { + Filter *TableFilter + Table Table +} - for _, r := range m.Rules { +func (m Mappings) chosen(tables []Table) map[string]*Rule { + chosen := make(map[string]*Rule, len(tables)) + + for i := range m.Rules { + r := &m.Rules[i] if r.Type != ruleSelection { continue } for _, t := range tables { - if !MatchPattern(r.Locator.Schema, t.Schema) || !MatchPattern(r.Locator.Table, t.Name) { + if !r.selects(t) { continue } - chosen[t.Key()] = r.Action != actExclude + if r.Action == actExclude { + delete(chosen, t.Key()) + } else { + chosen[t.Key()] = r + } } } + return chosen +} + +func (r Rule) selects(t Table) bool { + return MatchPattern(r.Locator.Schema, t.Schema) && MatchPattern(r.Locator.Table, t.Name) +} + +// Select returns the source tables chosen by the selection rules, in source order. +func (m Mappings) Select(tables []Table) []Table { + chosen := m.chosen(tables) out := make([]Table, 0, len(chosen)) for _, t := range tables { - if chosen[t.Key()] { + if _, ok := chosen[t.Key()]; ok { out = append(out, t) } } @@ -213,9 +425,43 @@ func (m Mappings) Select(tables []Table) []Table { return out } +// SelectFiltered is Select plus each table's compiled source filter; tables with a higher load-order come first. +func (m Mappings) SelectFiltered(tables []Table) ([]SelectedTable, error) { + chosen := m.chosen(tables) + out := make([]SelectedTable, 0, len(chosen)) + + for _, t := range tables { + r, ok := chosen[t.Key()] + if !ok { + continue + } + + var fs []RuleFilter + if r.Action == actInclude { + fs = r.Filters + } + + f, err := newTableFilter(t, fs) + if err != nil { + return nil, err + } + + out = append(out, SelectedTable{Table: t, Filter: f}) + } + + order := func(st SelectedTable) int { + n, _ := strconv.Atoi(string(chosen[st.Table.Key()].LoadOrder)) + + return n + } + sort.SliceStable(out, func(i, j int) bool { return order(out[i]) > order(out[j]) }) + + return out, nil +} + // Plan maps a selected source table to its target shape. overrideSchema (TargetMetadata.TargetSchema) wins // over schema renames when set. -func (m Mappings) Plan(t Table, src Family, overrideSchema string) TargetTable { +func (m Mappings) Plan(t Table, src Family, overrideSchema string) (TargetTable, error) { out := TargetTable{ SourceFamily: src, Schema: t.Schema, Name: t.Name, SourceSchema: t.Schema, SourceName: t.Name, } @@ -241,53 +487,9 @@ func (m Mappings) Plan(t Table, src Family, overrideSchema string) TargetTable { out.Schema = overrideSchema } - out.Columns, out.PK = m.planColumns(t) - - return out -} - -func (m Mappings) planColumns(t Table) ([]TargetColumn, []string) { - cols := make([]TargetColumn, 0, len(t.Columns)) - rename := make(map[string]string, len(t.Columns)) - - for i, c := range t.Columns { - name, removed := c.Name, false - - for _, r := range m.Rules { - if r.Type != ruleTransformation || r.Target != targetColumn || !m.columnRuleApplies(r, t, c.Name) { - continue - } - - if r.Action == actRemoveColumn { - removed = true - - break - } - - name = applyNameAction(r.Action, name, r.Value, r.Old) - } - - if removed { - continue - } - - rename[c.Name] = name - c.Name = name - cols = append(cols, TargetColumn{Column: c, Source: i}) - } - - pk := make([]string, 0, len(t.PK)) - - for _, p := range t.PK { - if n, ok := rename[p]; ok { - pk = append(pk, n) - } + if err := m.planColumns(t, &out); err != nil { + return out, err } - return cols, pk -} - -func (m Mappings) columnRuleApplies(r Rule, t Table, col string) bool { - return MatchPattern(r.Locator.Schema, t.Schema) && MatchPattern(r.Locator.Table, t.Name) && - MatchPattern(r.Locator.Column, col) + return out, m.definePrimaryKey(t, &out) } diff --git a/pkgs/dmsengine/mapping_test.go b/pkgs/dmsengine/mapping_test.go index 83925277c3..52333c50cb 100644 --- a/pkgs/dmsengine/mapping_test.go +++ b/pkgs/dmsengine/mapping_test.go @@ -162,7 +162,8 @@ func TestMappingsPlan(t *testing.T) { m, err := dmsengine.ParseMappings(doc) require.NoError(t, err) - got := m.Plan(sampleTables()[0], dmsengine.FamilyMySQL, tt.override) + got, perr := m.Plan(sampleTables()[0], dmsengine.FamilyMySQL, tt.override) + require.NoError(t, perr) assert.Equal(t, tt.wantSchema, got.Schema) assert.Equal(t, tt.wantTable, got.Name) diff --git a/pkgs/dmsengine/model.go b/pkgs/dmsengine/model.go index 701c8a5ca8..86a5305d93 100644 --- a/pkgs/dmsengine/model.go +++ b/pkgs/dmsengine/model.go @@ -58,6 +58,7 @@ func FamilyOf(engine string) Family { type Column struct { Name string Native string + DMS DMSType Kind Kind Nullable bool } @@ -136,33 +137,66 @@ const ( // TargetColumn is a column on the target; Source indexes the originating source column. type TargetColumn struct { + Expr *Expr Column Source int + // Computed marks an add-column column evaluated from Expr; Before marks a before-image column. + Computed bool + Before bool } // TargetTable is a source table after mapping rules. type TargetTable struct { - Schema string - Name string - SourceSchema string - SourceName string - Columns []TargetColumn - PK []string - SourceFamily Family + Schema string + Name string + SourceSchema string + SourceName string + KeyName string + Columns []TargetColumn + PK []string + SourceFamily Family + KeyUnique bool + HasBeforeImage bool } // Key is "schema.table" on the target. func (t TargetTable) Key() string { return t.Schema + "." + t.Name } -// Project selects and orders a source row's values for the target. +// ProjectCtx carries what computed and before-image columns need to fill in one row. +type ProjectCtx struct { + Time time.Time + Now time.Time + Op Op + Position string + Before Row + Full bool +} + +// Project selects and orders a full-load source row's values for the target. func (t TargetTable) Project(r Row) Row { + return t.ProjectWith(r, ProjectCtx{Full: true, Now: time.Now()}) +} + +// ProjectWith is Project for a change row: r is the row image, pc.Before feeds the before-image columns. +func (t TargetTable) ProjectWith(r Row, pc ProjectCtx) Row { if r == nil { return nil } + ec := &EvalCtx{Row: r, Op: pc.Op, Time: pc.Time, Now: pc.Now, Position: pc.Position, Full: pc.Full} out := make(Row, len(t.Columns)) + for i, c := range t.Columns { - if c.Source < len(r) { + switch { + case c.Computed: + if c.Expr != nil { + out[i] = c.DMS.Cast(c.Expr.Eval(ec)) + } + case c.Before: + if pc.Before != nil && c.Source < len(pc.Before) && !pc.Full { + out[i] = pc.Before[c.Source] + } + case c.Source < len(r): out[i] = r[c.Source] } } diff --git a/pkgs/dmsengine/mysql_source.go b/pkgs/dmsengine/mysql_source.go index 44150fd4c2..79a2f4340b 100644 --- a/pkgs/dmsengine/mysql_source.go +++ b/pkgs/dmsengine/mysql_source.go @@ -147,6 +147,11 @@ func (s *mysqlSource) attachKeys(ctx context.Context, tables []Table, index map[ } func (s *mysqlSource) Scan(ctx context.Context, t Table, fn func(Row) error) error { + return s.ScanWhere(ctx, t, nil, fn) +} + +// ScanWhere is Scan restricted by a selection-rule filter, evaluated by the database. +func (s *mysqlSource) ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error { names := make([]string, len(t.Columns)) for i, c := range t.Columns { names[i] = QuoteIdent(FamilyMySQL, c.Name) @@ -155,7 +160,13 @@ func (s *mysqlSource) Scan(ctx context.Context, t Table, fn func(Row) error) err q := stmt("SELECT ", strings.Join(names, ", "), " FROM ", QuoteIdent(FamilyMySQL, t.Schema), ".", QuoteIdent(FamilyMySQL, t.Name)) - rows, err := s.db.QueryContext(ctx, q) + var args []any + + if where, wargs := f.SQL(FamilyMySQL, 1); where != "" { + q, args = q+" WHERE "+where, wargs + } + + rows, err := s.db.QueryContext(ctx, q, args...) if err != nil { return err } diff --git a/pkgs/dmsengine/pg_source.go b/pkgs/dmsengine/pg_source.go index e7eeea4e5d..c46e090052 100644 --- a/pkgs/dmsengine/pg_source.go +++ b/pkgs/dmsengine/pg_source.go @@ -135,6 +135,11 @@ func (s *pgSource) Tables(ctx context.Context) ([]Table, error) { } func (s *pgSource) Scan(ctx context.Context, t Table, fn func(Row) error) error { + return s.ScanWhere(ctx, t, nil, fn) +} + +// ScanWhere is Scan restricted by a selection-rule filter, evaluated by the database. +func (s *pgSource) ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error { names := make([]string, len(t.Columns)) for i, c := range t.Columns { @@ -149,7 +154,13 @@ func (s *pgSource) Scan(ctx context.Context, t Table, fn func(Row) error) error q := stmt("SELECT ", strings.Join(names, ", "), " FROM ", QuoteIdent(FamilyPostgres, t.Schema), ".", QuoteIdent(FamilyPostgres, t.Name)) - rows, err := s.db.QueryContext(ctx, q) + var args []any + + if where, wargs := f.SQL(FamilyPostgres, 1); where != "" { + q, args = q+" WHERE "+where, wargs + } + + rows, err := s.db.QueryContext(ctx, q, args...) if err != nil { return err } @@ -334,7 +345,7 @@ func parseDecoded(lsn, data string, tables func(string, string) (Table, bool)) ( c := Change{Schema: t.Schema, Table: t.Name} switch opName { - case "INSERT": + case opWordInsert: c.Op, c.After = OpInsert, tupleRow(parseTuple(body), t) case "DELETE": c.Op, c.Before = OpDelete, tupleRow(parseTuple(body), t) diff --git a/pkgs/dmsengine/plan.go b/pkgs/dmsengine/plan.go new file mode 100644 index 0000000000..861c015343 --- /dev/null +++ b/pkgs/dmsengine/plan.go @@ -0,0 +1,232 @@ +package dmsengine + +import ( + "fmt" + "slices" + "strings" +) + +func (m Mappings) tableRules(t Table, target string) []Rule { + var out []Rule + + for _, r := range m.Rules { + if r.Type == ruleTransformation && r.Target == target && MatchPattern(r.Locator.Schema, t.Schema) && + MatchPattern(r.Locator.Table, t.Name) { + out = append(out, r) + } + } + + return out +} + +func columnMatches(r Rule, c Column) bool { + if !MatchPattern(r.Locator.Column, c.Name) { + return false + } + + if r.Locator.DataType == "" { + return true + } + + return strings.EqualFold(r.Locator.DataType, c.DMS.Name) || + (c.DMS.Name == "" && strings.EqualFold(r.Locator.DataType, dmsTypeOf(c.Native))) +} + +type colPlan struct { + out []TargetColumn + rename map[string]string + rules []Rule + include bool +} + +func (m Mappings) planColumns(t Table, out *TargetTable) error { + cp := colPlan{rules: m.tableRules(t, targetColumn), rename: make(map[string]string, len(t.Columns))} + + for _, r := range cp.rules { + cp.include = cp.include || r.Action == actIncludeColumn + } + + for i, c := range t.Columns { + tc, keep, err := cp.sourceColumn(i, c) + if err != nil { + return err + } + + if keep { + cp.rename[c.Name] = tc.Name + cp.out = append(cp.out, tc) + } + } + + if err := cp.addColumns(t); err != nil { + return err + } + + cp.beforeImages(t) + + if err := checkDuplicates(cp.out, out.Key()); err != nil { + return err + } + + out.Columns = cp.out + out.PK = make([]string, 0, len(t.PK)) + + for _, p := range t.PK { + if n, ok := cp.rename[p]; ok { + out.PK = append(out.PK, n) + } + } + + for _, c := range cp.out { + out.HasBeforeImage = out.HasBeforeImage || c.Before + } + + return nil +} + +func checkDuplicates(cols []TargetColumn, table string) error { + seen := make(map[string]bool, len(cols)) + + for _, c := range cols { + if seen[c.Name] { + return fmt.Errorf("%w: column %q appears twice on %s", ErrMapping, c.Name, table) + } + + seen[c.Name] = true + } + + return nil +} + +func (cp *colPlan) sourceColumn(i int, c Column) (TargetColumn, bool, error) { + name, included, removed := c.Name, !cp.include, false + tc := TargetColumn{Column: c, Source: i} + + for _, r := range cp.rules { + if !columnMatches(r, c) { + continue + } + + switch r.Action { + case actRemoveColumn: + removed = true + case actIncludeColumn: + included = true + case actChangeType: + dt, err := r.DataType.parse() + if err != nil { + return tc, false, err + } + + tc.Native, tc.Kind, tc.DMS = dt.Name, dt.Kind(), dt + case actAddColumn, actBeforeImage: + default: + name = applyNameAction(r.Action, name, r.Value, r.Old) + } + } + + tc.Name = name + + return tc, included && !removed, nil +} + +func (cp *colPlan) addColumns(t Table) error { + for _, r := range cp.rules { + if r.Action != actAddColumn { + continue + } + + dt, err := r.DataType.parse() + if err != nil { + return err + } + + col := TargetColumn{Column: dt.Column(r.Value), Source: -1, Computed: true} + + if strings.TrimSpace(r.Expression) != "" { + e, eerr := CompileExpr(r.Expression, t) + if eerr != nil { + return fmt.Errorf("add-column %q on %s: %w", r.Value, t.Key(), eerr) + } + + col.Expr = e + } + + cp.out = append(cp.out, col) + } + + return nil +} + +func isLOB(c Column) bool { + switch c.DMS.Name { + case tBlob, "clob", tNclob: + return true + } + + return slices.Contains([]string{ + tBlob, tTinyblob, tMediumblob, "longblob", "text", "tinytext", "mediumtext", "longtext", "json", "jsonb", + "bytea", + }, parseNative(c.Native).base) +} + +func (cp *colPlan) beforeImages(t Table) { + for _, r := range cp.rules { + if r.Action != actBeforeImage { + continue + } + + def := r.imageDef() + + for i, c := range t.Columns { + if _, kept := cp.rename[c.Name]; !kept || !imageWanted(def.Filter, t, c) { + continue + } + + col := c + col.Name, col.Nullable = *def.Prefix+c.Name+def.Suffix, true + cp.out = append(cp.out, TargetColumn{Column: col, Source: i, Before: true}) + } + } +} + +func imageWanted(filter string, t Table, c Column) bool { + if isLOB(c) { + return false + } + + return filter != imageFilterPK || slices.Contains(t.PK, c.Name) +} + +func (m Mappings) definePrimaryKey(t Table, out *TargetTable) error { + for _, r := range m.tableRules(t, targetTable) { + if r.Action != actDefinePK || r.Locator.Schema != t.Schema || r.Locator.Table != t.Name { + continue + } + + cols := make([]string, 0, len(r.PrimaryKey.Columns)) + + for _, name := range r.PrimaryKey.Columns { + target, ok := out.columnNamed(t, name) + if !ok { + return fmt.Errorf("%w: primary-key-def column %q does not exist on %s", ErrMapping, name, t.Key()) + } + + cols = append(cols, target) + } + + out.PK, out.KeyName, out.KeyUnique = cols, r.PrimaryKey.Name, r.PrimaryKey.Origin == pkOriginUnique + } + + return nil +} + +func (tt TargetTable) columnNamed(t Table, name string) (string, bool) { + for _, c := range tt.Columns { + if c.Name == name || (!c.Computed && !c.Before && t.Columns[c.Source].Name == name) { + return c.Name, true + } + } + + return "", false +} diff --git a/pkgs/dmsengine/rules_test.go b/pkgs/dmsengine/rules_test.go new file mode 100644 index 0000000000..9b2b8214ae --- /dev/null +++ b/pkgs/dmsengine/rules_test.go @@ -0,0 +1,283 @@ +package dmsengine_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func employeeTable() dmsengine.Table { + return dmsengine.Table{Schema: "test", Name: "employee", PK: []string{"empid"}, Columns: []dmsengine.Column{ + {Name: "empid", Native: "int", Kind: dmsengine.KindInt}, + {Name: "dept", Native: "varchar(20)", Kind: dmsengine.KindString, Nullable: true}, + {Name: "salary", Native: "decimal(10,2)", Kind: dmsengine.KindDecimal, Nullable: true}, + {Name: "started", Native: "date", Kind: dmsengine.KindTime, Nullable: true}, + {Name: "notes", Native: "text", Kind: dmsengine.KindString, Nullable: true}, + }} +} + +func filterRule(filters string) string { + return `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1", + "object-locator":{"schema-name":"test","table-name":"employee"},"rule-action":"include","filters":[` + + filters + `]}]}` +} + +func srcFilter(col, conds string) string { + return `{"filter-type":"source","column-name":"` + col + `","filter-conditions":[` + conds + `]}` +} + +func TestSelectionFilterMatch(t *testing.T) { + t.Parallel() + + row := func(id, dept, salary, started string) dmsengine.Row { + r := dmsengine.Row{id, dept, salary, started, nil} + if dept == "" { + r[1] = nil + } + + return r + } + + tests := []struct { + name string + filters string + rows []dmsengine.Row + want []bool + }{ + { + name: "gte", + filters: srcFilter("empid", `{"filter-operator":"gte","value":"100"}`), + rows: []dmsengine.Row{ + row("99", "a", "1", "2020-01-01"), + row("100", "a", "1", "2020-01-01"), + row("1000", "a", "1", "2020-01-01"), + }, + want: []bool{false, true, true}, + }, + { + name: "or of conditions in one filter", + filters: srcFilter( + "empid", + `{"filter-operator":"lte","value":"10"},{"filter-operator":"between","start-value":"50",`+ + `"end-value":"75"},{"filter-operator":"gte","value":"100"}`, + ), + rows: []dmsengine.Row{ + row("5", "a", "1", "2020-01-01"), + row("20", "a", "1", "2020-01-01"), + row("75", "a", "1", "2020-01-01"), + row("120", "a", "1", "2020-01-01"), + }, + want: []bool{true, false, true, true}, + }, + { + name: "and of filters", + filters: srcFilter("empid", `{"filter-operator":"ste","value":"100"}`) + "," + + srcFilter("dept", `{"filter-operator":"eq","value":"tech"}`), + rows: []dmsengine.Row{ + row("50", "tech", "1", "2020-01-01"), + row("50", "ops", "1", "2020-01-01"), + row("500", "tech", "1", "2020-01-01"), + }, + want: []bool{true, false, false}, + }, + { + name: "null and notnull", + filters: srcFilter("dept", `{"filter-operator":"null"}`), + rows: []dmsengine.Row{row("1", "", "1", "2020-01-01"), row("2", "ops", "1", "2020-01-01")}, + want: []bool{true, false}, + }, + { + name: "notbetween skips null", + filters: srcFilter("salary", `{"filter-operator":"notbetween","start-value":"50","end-value":"75"}`), + rows: []dmsengine.Row{ + row("1", "a", "49.5", "2020-01-01"), + row("2", "a", "60", "2020-01-01"), + {"3", "a", nil, nil, nil}, + }, + want: []bool{true, false, false}, + }, + { + name: "noteq numeric compare not text", + filters: srcFilter("empid", `{"filter-operator":"noteq","value":"9"}`), + rows: []dmsengine.Row{row("9", "a", "1", "2020-01-01"), row("10", "a", "1", "2020-01-01")}, + want: []bool{false, true}, + }, + { + name: "date", + filters: srcFilter("started", `{"filter-operator":"gte","value":"2002-01-01"}`), + rows: []dmsengine.Row{row("1", "a", "1", "2001-12-31"), row("2", "a", "1", "2002-01-01")}, + want: []bool{false, true}, + }, + { + name: "negative with null matches nothing", + filters: srcFilter("empid", `{"filter-operator":"noteq","value":"1"},{"filter-operator":"null"}`), + rows: []dmsengine.Row{row("2", "a", "1", "2020-01-01")}, + want: []bool{false}, + }, + { + name: "inclusive negative and null drop null rows", + filters: srcFilter( + "dept", + `{"filter-operator":"eq","value":"a"},{"filter-operator":"noteq","value":"b"},`+ + `{"filter-operator":"null"}`, + ), + rows: []dmsengine.Row{row("1", "c", "1", "2020-01-01"), row("2", "", "1", "2020-01-01")}, + want: []bool{true, false}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(filterRule(tt.filters)) + require.NoError(t, err) + + sel, err := m.SelectFiltered([]dmsengine.Table{employeeTable()}) + require.NoError(t, err) + require.Len(t, sel, 1) + + for i, r := range tt.rows { + assert.Equal(t, tt.want[i], sel[0].Filter.Match(r), "row %d", i) + } + }) + } +} + +func TestSelectionFilterSQL(t *testing.T) { + t.Parallel() + + filters := srcFilter( + "empid", + `{"filter-operator":"lte","value":"10"},{"filter-operator":"between","start-value":"50","end-value":"75"}`, + ) + + "," + srcFilter( + "dept", + `{"filter-operator":"notnull"}`, + ) + + m, err := dmsengine.ParseMappings(filterRule(filters)) + require.NoError(t, err) + + sel, err := m.SelectFiltered([]dmsengine.Table{employeeTable()}) + require.NoError(t, err) + + where, args := sel[0].Filter.SQL(dmsengine.FamilyPostgres, 1) + assert.Equal(t, `("empid" <= $1 OR "empid" BETWEEN $2 AND $3) AND ("dept" IS NOT NULL)`, where) + assert.Equal(t, []any{"10", "50", "75"}, args) + + where, _ = sel[0].Filter.SQL(dmsengine.FamilyMySQL, 1) + assert.Equal(t, "(`empid` <= ? OR `empid` BETWEEN ? AND ?) AND (`dept` IS NOT NULL)", where) +} + +func TestParseMappingsRuleErrors(t *testing.T) { + t.Parallel() + + sel := `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"%",` + + `"table-name":"%"},"rule-action":"include"}` + tests := []struct { + name string + rule string + }{ + { + "filter on wildcard table", + `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"test",` + + `"table-name":"emp%"},"rule-action":"include","filters":[` + srcFilter( + "a", + `{"filter-operator":"eq","value":"1"}`, + ) + `]}`, + }, + { + "bad operator", + `{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"test",` + + `"table-name":"employee"},"rule-action":"include","filters":[` + srcFilter( + "a", + `{"filter-operator":"like","value":"1"}`, + ) + `]}`, + }, + { + "add-column without data-type", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%"},"value":"x"}`, + }, + { + "unsupported data type", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%","column-name":"%"},` + + `"data-type":{"type":"map"}}`, + }, + { + "define-primary-key wildcard", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"define-primary-key",` + + `"rule-target":"table","object-locator":{"schema-name":"%","table-name":"t"},` + + `"primary-key-def":{"name":"pk","columns":["a"]}}`, + }, + { + "before image both empty", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column",` + + `"object-locator":{"schema-name":"%","table-name":"%"},` + + `"before-image-def":{"column-prefix":"","column-suffix":""}}`, + }, + { + "data masking", + `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"data-masking-hash-mask",` + + `"rule-target":"column","object-locator":{"schema-name":"%","table-name":"%","column-name":"%"}}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := dmsengine.ParseMappings(`{"rules":[` + sel + `,` + tt.rule + `]}`) + require.ErrorIs(t, err, dmsengine.ErrMapping) + }) + } +} + +func TestMatchPatternWildcards(t *testing.T) { + t.Parallel() + + tests := []struct { + pattern, s string + want bool + }{ + {"emp_", "empx", true}, + {"emp_", "emp", false}, + {"emp_", "empxx", false}, + {"e_p%", "excess", false}, + {"e_p%", "exp_anything", true}, + {"%", "", true}, + {"a.c", "abc", false}, + {"a%c", "a\nc", true}, + } + + for _, tt := range tests { + t.Run(tt.pattern+"/"+tt.s, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, dmsengine.MatchPattern(tt.pattern, tt.s)) + }) + } +} + +func TestSelectLoadOrderAndExplicit(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(`{"rules":[ + {"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app", + "table-name":"items"},"rule-action":"include","load-order":"1"}, + {"rule-type":"selection","rule-id":"2","rule-name":"2","object-locator":{"schema-name":"app", + "table-name":"orders"},"rule-action":"explicit","load-order":"2"}]}`) + require.NoError(t, err) + + sel, err := m.SelectFiltered(twoTables().tables) + require.NoError(t, err) + require.Len(t, sel, 2) + assert.Equal(t, "orders", sel[0].Table.Name) + assert.Equal(t, "items", sel[1].Table.Name) +} diff --git a/pkgs/dmsengine/s3_source.go b/pkgs/dmsengine/s3_source.go new file mode 100644 index 0000000000..cb13998777 --- /dev/null +++ b/pkgs/dmsengine/s3_source.go @@ -0,0 +1,518 @@ +package dmsengine + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strconv" + "strings" + "time" +) + +// Errors reported by the S3 source. +var ( + ErrS3Source = errors.New("invalid S3 source") + errS3SourceBucket = fmt.Errorf("%w: S3Settings.BucketName is required", ErrS3Source) + errS3SourceDef = fmt.Errorf("%w: an ExternalTableDefinition is required", ErrS3Source) + errS3SourceCDCPath = fmt.Errorf("%w: S3Settings.CdcPath is required to capture changes", ErrS3Source) + errS3SourceParquet = fmt.Errorf("%w: Parquet source files are not supported", ErrS3Source) + errS3SourceDelim = fmt.Errorf("%w: CsvDelimiter must be a single character", ErrS3Source) + errS3SourceNoStore = fmt.Errorf("%w: %w: S3", ErrS3Source, ErrNotConfigured) +) + +const ( + s3SourceLoadPrefix = "load" + s3SourceCSVSuffix = ".csv" + s3SourceMinCDCCells = 3 +) + +// jsonScalar reads a JSON string, number or boolean as its text; the external table definition quotes everything. +type jsonScalar string + +func (s *jsonScalar) UnmarshalJSON(b []byte) error { + var v any + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + switch x := v.(type) { + case nil: + *s = "" + case string: + *s = jsonScalar(x) + default: + *s = jsonScalar(strings.Trim(string(b), `"`)) + } + + return nil +} + +func (s jsonScalar) bool() bool { return strings.EqualFold(strings.TrimSpace(string(s)), "true") } + +func (s jsonScalar) int() int { + n, _ := strconv.Atoi(strings.TrimSpace(string(s))) + + return n +} + +type s3ColumnDef struct { + Name string `json:"ColumnName"` + Type string `json:"ColumnType"` + Length jsonScalar `json:"ColumnLength"` + Precision jsonScalar `json:"ColumnPrecision"` + Scale jsonScalar `json:"ColumnScale"` + Nullable jsonScalar `json:"ColumnNullable"` + IsPK jsonScalar `json:"ColumnIsPk"` + DateFormat string `json:"ColumnDateFormat"` +} + +type s3TableDef struct { + Name string `json:"TableName"` + Path string `json:"TablePath"` + Owner string `json:"TableOwner"` + ColumnsTotal jsonScalar `json:"TableColumnsTotal"` + Columns []s3ColumnDef `json:"TableColumns"` +} + +type s3SourceOptions struct { + Rfc4180 *bool `json:"Rfc4180"` + BucketName string `json:"BucketName"` + BucketFolder string `json:"BucketFolder"` + CdcPath string `json:"CdcPath"` + CsvDelimiter string `json:"CsvDelimiter"` + CsvRowDelimiter string `json:"CsvRowDelimiter"` + CsvNullValue string `json:"CsvNullValue"` + DataFormat string `json:"DataFormat"` + ExternalTableDef string `json:"ExternalTableDefinition"` + IgnoreHeaderRows int `json:"IgnoreHeaderRows"` +} + +type s3Source struct { + store ObjectStore + paths map[string]string + dates map[string][]string + opts s3SourceOptions + tables []Table + poll time.Duration + delim rune +} + +func newS3Source(ep EndpointConfig, store ObjectStore) (Source, error) { + if store == nil { + return nil, errS3SourceNoStore + } + + var o s3SourceOptions + if ep.S3Settings != "" { + if err := json.Unmarshal([]byte(ep.S3Settings), &o); err != nil { + return nil, fmt.Errorf("%w: S3Settings: %w", ErrS3Source, err) + } + } + + if o.BucketName == "" { + return nil, errS3SourceBucket + } + + if strings.EqualFold(o.DataFormat, "parquet") { + return nil, errS3SourceParquet + } + + s := &s3Source{store: store, opts: o, poll: ep.PollInterval, delim: ',', paths: map[string]string{}, + dates: map[string][]string{}} + + if o.CsvDelimiter != "" { + rs := []rune(o.CsvDelimiter) + if len(rs) != 1 { + return nil, errS3SourceDelim + } + + s.delim = rs[0] + } + + def := ep.ExternalTableDefinition + if def == "" { + def = o.ExternalTableDef + } + + if err := s.loadDefinition(def); err != nil { + return nil, err + } + + if s.poll <= 0 { + s.poll = defaultPGPoll + } + + return s, nil +} + +func (s *s3Source) loadDefinition(def string) error { + if strings.TrimSpace(def) == "" { + return errS3SourceDef + } + + var doc struct { + Tables []s3TableDef `json:"Tables"` + } + + if err := json.Unmarshal([]byte(def), &doc); err != nil { + return fmt.Errorf("%w: ExternalTableDefinition: %w", ErrS3Source, err) + } + + for _, td := range doc.Tables { + t, err := td.table() + if err != nil { + return err + } + + s.tables = append(s.tables, t) + s.paths[t.Key()] = td.Path + + fm := make([]string, len(td.Columns)) + for i, c := range td.Columns { + fm[i] = c.DateFormat + } + + s.dates[t.Key()] = fm + } + + return nil +} + +func (td s3TableDef) table() (Table, error) { + if td.Name == "" || len(td.Columns) == 0 { + return Table{}, fmt.Errorf("%w: a table needs TableName and TableColumns", ErrS3Source) + } + + if td.ColumnsTotal != "" && td.ColumnsTotal.int() != len(td.Columns) { + return Table{}, fmt.Errorf("%w: %s: TableColumnsTotal %s does not match its %d columns", + ErrS3Source, td.Name, td.ColumnsTotal, len(td.Columns)) + } + + t := Table{Schema: td.Owner, Name: td.Name} + + for _, c := range td.Columns { + kind := strings.ToLower(c.Type) + if kind == "byte" { + kind = tBytes + } + + dt, err := ParseDMSType(kind) + if err != nil { + return t, fmt.Errorf("%w: %s.%s: %w", ErrS3Source, td.Name, c.Name, err) + } + + dt.Length, dt.Precision, dt.Scale = c.Length.int(), c.Precision.int(), c.Scale.int() + col := dt.Column(c.Name) + col.Nullable = c.Nullable.bool() + t.Columns = append(t.Columns, col) + + if c.IsPK.bool() { + t.PK = append(t.PK, c.Name) + } + } + + return t, nil +} + +func (s *s3Source) Ping(ctx context.Context) error { + return s.store.CheckBucket(ctx, s.opts.BucketName) +} + +func (s *s3Source) Close() error { return nil } + +func (s *s3Source) Release(context.Context) error { return nil } + +func (s *s3Source) Tables(context.Context) ([]Table, error) { return s.tables, nil } + +func (s *s3Source) Position(context.Context) (string, error) { return "", nil } + +func (s *s3Source) key(parts ...string) string { + all := []string{} + if f := strings.Trim(s.opts.BucketFolder, "/"); f != "" { + all = append(all, f) + } + + for _, p := range parts { + if p = strings.Trim(p, "/"); p != "" { + all = append(all, p) + } + } + + return strings.Join(all, "/") +} + +func (s *s3Source) tablePath(t Table) string { + if p := s.paths[t.Key()]; p != "" { + return p + } + + return t.Schema + "/" + t.Name +} + +func (s *s3Source) Scan(ctx context.Context, t Table, fn func(Row) error) error { + dir := s.key(s.tablePath(t)) + "/" + + keys, err := s.store.ListObjects(ctx, s.opts.BucketName, dir) + if err != nil { + return err + } + + for _, k := range keys { + base := strings.ToLower(strings.TrimPrefix(k, dir)) + if strings.Contains(base, "/") || !strings.HasPrefix(base, s3SourceLoadPrefix) || + !strings.HasSuffix(base, s3SourceCSVSuffix) { + continue + } + + recs, rerr := s.records(ctx, k) + if rerr != nil { + return rerr + } + + for i, rec := range recs { + row, rowErr := s.fullLoadRow(t, rec) + if rowErr != nil { + return fmt.Errorf("%s record %d: %w", k, i+1, rowErr) + } + + if err = fn(row); err != nil { + return err + } + } + } + + return nil +} + +func (s *s3Source) records(ctx context.Context, key string) ([][]string, error) { + body, err := s.store.GetObject(ctx, s.opts.BucketName, key) + if err != nil { + return nil, err + } + + if rd := s.opts.CsvRowDelimiter; rd != "" && rd != "\n" && rd != "\r\n" { + body = bytes.ReplaceAll(body, []byte(rd), []byte("\n")) + } + + r := csv.NewReader(bytes.NewReader(body)) + r.Comma, r.FieldsPerRecord, r.LazyQuotes = s.delim, -1, s.opts.Rfc4180 != nil && !*s.opts.Rfc4180 + + var out [][]string + + for { + rec, rerr := r.Read() + if errors.Is(rerr, io.EOF) { + break + } + + if rerr != nil { + return nil, fmt.Errorf("%s: %w", key, rerr) + } + + out = append(out, rec) + } + + return out[min(s.opts.IgnoreHeaderRows, len(out)):], nil +} + +func isInsertOp(s string) bool { + return strings.EqualFold(s, "I") || strings.EqualFold(s, opWordInsert) +} + +func (s *s3Source) fullLoadRow(t Table, rec []string) (Row, error) { + if len(rec) == len(t.Columns)+1 && isInsertOp(rec[0]) { + rec = rec[1:] + } + + return s.row(t, rec) +} + +func (s *s3Source) row(t Table, rec []string) (Row, error) { + if len(rec) != len(t.Columns) { + return nil, fmt.Errorf( + "%w: %s has %d columns, the record has %d", + ErrS3Source, + t.Key(), + len(t.Columns), + len(rec), + ) + } + + fm := s.dates[t.Key()] + row := make(Row, len(rec)) + + for i, f := range rec { + switch { + case f == s.opts.CsvNullValue: + row[i] = nil + case t.Columns[i].Kind == KindBytes: + row[i] = []byte(f) + default: + row[i] = normalizeDate(f, fm[i], t.Columns[i].DMS.Name) + } + } + + return row, nil +} + +func dateTokens() *strings.Replacer { + return strings.NewReplacer( + "YYYY", + "2006", + "MM", + "01", + "dd", + "02", + "HH", + "15", + "mm", + "04", + "ss", + "05", + ".F", + ".999999999", + ) +} + +// normalizeDate rewrites a ColumnDateFormat value to the ISO form databases accept. +func normalizeDate(v, format, dmsType string) string { + if format == "" { + return v + } + + t, err := time.Parse(dateTokens().Replace(format), v) + if err != nil { + return v + } + + switch dmsType { + case tDate: + return t.Format("2006-01-02") + case tTime: + return t.Format("15:04:05") + } + + return t.Format("2006-01-02 15:04:05.999999999") +} + +func (s *s3Source) Stream( + ctx context.Context, start StreamStart, tables func(string, string) (Table, bool), fn func(Change) error, +) error { + if s.opts.CdcPath == "" { + return errS3SourceCDCPath + } + + last := start.Position + tick := time.NewTicker(s.poll) + + defer tick.Stop() + + for { + var err error + if last, err = s.drain(ctx, last, tables, fn); err != nil { + if ctx.Err() != nil { + return nil + } + + return err + } + + select { + case <-ctx.Done(): + return nil + case <-tick.C: + } + } +} + +// drain applies the CDC files that sort after last, in order, and returns the last key applied. +func (s *s3Source) drain( + ctx context.Context, last string, tables func(string, string) (Table, bool), fn func(Change) error, +) (string, error) { + dir := s.key(s.opts.CdcPath) + "/" + + keys, err := s.store.ListObjects(ctx, s.opts.BucketName, dir) + if err != nil { + return last, err + } + + for _, k := range keys { + base := strings.ToLower(path.Base(k)) + if k <= last || strings.Contains(strings.TrimPrefix(k, dir), "/") || + !strings.HasSuffix(base, s3SourceCSVSuffix) { + continue + } + + if err = s.applyFile(ctx, k, tables, fn); err != nil { + return last, err + } + + last = k + } + + return last, nil +} + +func (s *s3Source) applyFile( + ctx context.Context, key string, tables func(string, string) (Table, bool), fn func(Change) error, +) error { + recs, err := s.records(ctx, key) + if err != nil { + return err + } + + for i, rec := range recs { + c, ok, cerr := s.change(rec, tables) + if cerr != nil { + return fmt.Errorf("%s record %d: %w", key, i+1, cerr) + } + + if !ok { + continue + } + + c.Position, c.Time = key, time.Now().UTC() + if err = fn(c); err != nil { + return err + } + } + + return fn(Change{Commit: true, Position: key, Time: time.Now().UTC()}) +} + +// change reads one CDC record: operation, table name, schema name, then the row. +func (s *s3Source) change(rec []string, tables func(string, string) (Table, bool)) (Change, bool, error) { + if len(rec) < s3SourceMinCDCCells { + return Change{}, false, fmt.Errorf("%w: a CDC record needs operation, table, schema and data", ErrS3Source) + } + + t, ok := tables(rec[2], rec[1]) + if !ok { + return Change{}, false, nil + } + + row, err := s.row(t, rec[s3SourceMinCDCCells:]) + if err != nil { + return Change{}, false, err + } + + c := Change{Schema: t.Schema, Table: t.Name} + + switch strings.ToUpper(rec[0]) { + case "I", opWordInsert: + c.Op, c.After = OpInsert, row + case "U", "UPDATE": + c.Op, c.After = OpUpdate, row + case "D", "DELETE": + c.Op, c.Before = OpDelete, row + default: + return Change{}, false, fmt.Errorf("%w: unknown operation %q", ErrS3Source, rec[0]) + } + + return c, true, nil +} diff --git a/pkgs/dmsengine/s3_source_test.go b/pkgs/dmsengine/s3_source_test.go new file mode 100644 index 0000000000..3f49cd3ca1 --- /dev/null +++ b/pkgs/dmsengine/s3_source_test.go @@ -0,0 +1,307 @@ +package dmsengine_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +const employeeDefinition = `{ + "TableCount": "1", + "Tables": [{ + "TableName": "employee", "TablePath": "hr/employee/", "TableOwner": "hr", + "TableColumns": [ + {"ColumnName": "Id", "ColumnType": "INT8", "ColumnNullable": "false", "ColumnIsPk": "true"}, + {"ColumnName": "LastName", "ColumnType": "STRING", "ColumnLength": "20", "ColumnNullable": "true"}, + {"ColumnName": "HireDate", "ColumnType": "DATETIME", "ColumnScale": "3", "ColumnNullable": "true"}, + {"ColumnName": "Rate", "ColumnType": "NUMERIC", "ColumnPrecision": "5", "ColumnScale": "2", + "ColumnNullable": "true"} + ], + "TableColumnsTotal": "4" + }] +}` + +func s3SourceEndpoint(settings string) dmsengine.EndpointConfig { + return dmsengine.EndpointConfig{ + Engine: "s3", S3Settings: settings, ExternalTableDefinition: employeeDefinition, + PollInterval: time.Millisecond, + } +} + +func openS3Source(t *testing.T, objs *memObjects, settings string) dmsengine.Source { + t.Helper() + + src, err := dmsengine.NewOpener(dmsengine.Stores{Objects: objs}).OpenSource(t.Context(), s3SourceEndpoint(settings)) + require.NoError(t, err) + + return src +} + +func TestS3SourceTablesAndScan(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + settings string + objects map[string]string + want []dmsengine.Row + }{ + { + name: "plain files under the table path", + settings: `{"BucketName":"b"}`, + objects: map[string]string{ + "b/hr/employee/LOAD001.csv": "101,Smith,2014-06-04 10:00:00,12.50\n102,,2015-10-08 00:00:00,7\n", + "b/hr/employee/LOAD002.csv": "103,Jones,2017-03-13 00:00:00,\n", + "b/hr/employee/other.txt": "ignored", + "b/hr/department/LOAD1.csv": "ignored", + }, + want: []dmsengine.Row{ + {"101", "Smith", "2014-06-04 10:00:00", "12.50"}, + {"102", nil, "2015-10-08 00:00:00", "7"}, + {"103", "Jones", "2017-03-13 00:00:00", nil}, + }, + }, + { + name: "bucket folder, custom delimiters, header and null marker", + settings: `{"BucketName":"b","BucketFolder":"src","CsvDelimiter":"|","CsvRowDelimiter":";", + "IgnoreHeaderRows":1,"CsvNullValue":"\\N"}`, + objects: map[string]string{ + "b/src/hr/employee/LOAD001.csv": "Id|LastName|HireDate|Rate;101|\\N|2014-06-04 10:00:00|1.5;", + }, + want: []dmsengine.Row{{"101", nil, "2014-06-04 10:00:00", "1.5"}}, + }, + { + name: "leading operation column from a DMS S3 target", + settings: `{"BucketName":"b"}`, + objects: map[string]string{"b/hr/employee/LOAD001.csv": "I,101,\"Smith, Jr\",2014-06-04 10:00:00,1\n"}, + want: []dmsengine.Row{{"101", "Smith, Jr", "2014-06-04 10:00:00", "1"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + src := openS3Source(t, &memObjects{objs: tt.objects}, tt.settings) + tables, err := src.Tables(t.Context()) + require.NoError(t, err) + require.Len(t, tables, 1) + + tbl := tables[0] + assert.Equal(t, "hr.employee", tbl.Key()) + assert.Equal(t, []string{"Id"}, tbl.PK) + assert.Equal(t, "int8", tbl.Columns[0].DMS.Name) + assert.Equal(t, 20, tbl.Columns[1].DMS.Length) + assert.Equal(t, 2, tbl.Columns[3].DMS.Scale) + assert.False(t, tbl.Columns[0].Nullable) + + var got []dmsengine.Row + + require.NoError(t, src.Scan(t.Context(), tbl, func(r dmsengine.Row) error { + got = append(got, r) + + return nil + })) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestS3SourceCDCFiles(t *testing.T) { + t.Parallel() + + objs := &memObjects{objs: map[string]string{ + "b/chg/CDC00001.csv": "INSERT,employee,hr,101,Smith,2014-06-04 10:00:00,1\n" + + "U,employee,hr,101,Smith,2015-10-08 00:00:00,2\n", + "b/chg/CDC00003.csv": "DELETE,employee,hr,101,Smith,2015-10-08 00:00:00,2\nI,other,hr,1,x,,\n", + }} + src := openS3Source(t, objs, `{"BucketName":"b","CdcPath":"chg"}`) + + tables, err := src.Tables(t.Context()) + require.NoError(t, err) + + lookup := func(schema, name string) (dmsengine.Table, bool) { + if schema == "hr" && name == "employee" { + return tables[0], true + } + + return dmsengine.Table{}, false + } + + type seen struct { + op dmsengine.Op + pos string + row dmsengine.Row + } + + ctx, cancel := context.WithCancel(t.Context()) + got := make(chan seen, 16) + done := make(chan error, 1) + + go func() { + done <- src.Stream(ctx, dmsengine.StreamStart{}, lookup, func(c dmsengine.Change) error { + row := c.After + if c.Op == dmsengine.OpDelete { + row = c.Before + } + + got <- seen{op: c.Op, row: row, pos: c.Position} + + return nil + }) + }() + + want := []seen{ + { + op: dmsengine.OpInsert, + row: dmsengine.Row{"101", "Smith", "2014-06-04 10:00:00", "1"}, + pos: "chg/CDC00001.csv", + }, + { + op: dmsengine.OpUpdate, + row: dmsengine.Row{"101", "Smith", "2015-10-08 00:00:00", "2"}, + pos: "chg/CDC00001.csv", + }, + {pos: "chg/CDC00001.csv"}, + { + op: dmsengine.OpDelete, + row: dmsengine.Row{"101", "Smith", "2015-10-08 00:00:00", "2"}, + pos: "chg/CDC00003.csv", + }, + {pos: "chg/CDC00003.csv"}, + } + + for _, w := range want { + assert.Equal(t, w, <-got) + } + + // A file that sorts before the last one applied is skipped, one that sorts after is picked up. + require.NoError(t, objs.PutObject(t.Context(), "b", "chg/CDC00002.csv", []byte("I,employee,hr,9,late,,\n"))) + require.NoError(t, objs.PutObject(t.Context(), "b", "chg/CDC00004.csv", []byte("I,employee,hr,4,fresh,,\n"))) + + assert.Equal( + t, + seen{op: dmsengine.OpInsert, row: dmsengine.Row{"4", "fresh", nil, nil}, pos: "chg/CDC00004.csv"}, + <-got, + ) + assert.Equal(t, seen{pos: "chg/CDC00004.csv"}, <-got) + + cancel() + require.NoError(t, <-done) +} + +func TestS3SourceOpenErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + store dmsengine.ObjectStore + name string + ep dmsengine.EndpointConfig + }{ + { + name: "no bucket", + ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{}`, + ExternalTableDefinition: employeeDefinition, + }, + store: &memObjects{}, + }, + { + name: "no definition", + ep: dmsengine.EndpointConfig{Engine: "s3", S3Settings: `{"BucketName":"b"}`}, + store: &memObjects{}, + }, + {name: "parquet", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b","DataFormat":"parquet"}`, + ExternalTableDefinition: employeeDefinition, + }, store: &memObjects{}}, + {name: "bad type", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: `{"Tables":[{"TableName":"t","TableOwner":"o", + "TableColumns":[{"ColumnName":"c","ColumnType":"MAP"}]}]}`, + }, store: &memObjects{}}, + {name: "column total mismatch", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: `{"Tables":[{"TableName":"t","TableOwner":"o","TableColumnsTotal":"2", + "TableColumns":[{"ColumnName":"c","ColumnType":"INT4"}]}]}`, + }, store: &memObjects{}}, + {name: "no store", ep: dmsengine.EndpointConfig{ + Engine: "s3", + S3Settings: `{"BucketName":"b"}`, + ExternalTableDefinition: employeeDefinition, + }, store: nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := dmsengine.NewOpener(dmsengine.Stores{Objects: tt.store}).OpenSource(t.Context(), tt.ep) + require.Error(t, err) + }) + } +} + +func TestS3SourceStreamNeedsCdcPath(t *testing.T) { + t.Parallel() + + src := openS3Source(t, &memObjects{}, `{"BucketName":"b"}`) + + err := src.Stream(t.Context(), dmsengine.StreamStart{}, nil, func(dmsengine.Change) error { return nil }) + require.ErrorIs(t, err, dmsengine.ErrS3Source) +} + +func TestApplySecret(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + secret string + want dmsengine.EndpointConfig + wantErr bool + }{ + { + name: "numeric port", + secret: `{"username":"admin","password":"pw","host":"db.example","port":3306,"engine":"mysql","dbname":"ignored"}`, + want: dmsengine.EndpointConfig{User: "admin", Password: "pw", Server: "db.example", Port: 3306}, + }, + { + name: "string port as the guide shows", + secret: `{"username":"admin","password":"pw","port":"8190","host":"db.example"}`, + want: dmsengine.EndpointConfig{User: "admin", Password: "pw", Server: "db.example", Port: 8190}, + }, + {name: "not json", secret: `plain`, wantErr: true}, + {name: "no host", secret: `{"username":"a","password":"b","port":1}`, wantErr: true}, + {name: "no port", secret: `{"username":"a","password":"b","host":"h"}`, wantErr: true}, + {name: "bad port", secret: `{"username":"a","password":"b","host":"h","port":"x"}`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ep := dmsengine.EndpointConfig{Database: "appdb"} + err := dmsengine.ApplySecret(&ep, tt.secret) + + if tt.wantErr { + require.ErrorIs(t, err, dmsengine.ErrSecret) + + return + } + + require.NoError(t, err) + + tt.want.Database = "appdb" + assert.Equal(t, tt.want, ep) + }) + } +} diff --git a/pkgs/dmsengine/secret.go b/pkgs/dmsengine/secret.go new file mode 100644 index 0000000000..32624a9345 --- /dev/null +++ b/pkgs/dmsengine/secret.go @@ -0,0 +1,57 @@ +package dmsengine + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" +) + +// ErrSecret reports a Secrets Manager secret that cannot authenticate an endpoint. +var ErrSecret = errors.New("invalid endpoint secret") + +// ApplySecret fills the endpoint's credentials from a Secrets Manager secret string. DMS reads username, password, +// host and port; other members are ignored ("Using secrets to access AWS Database Migration Service endpoints"). +func ApplySecret(ep *EndpointConfig, secret string) error { + var raw struct { + Username string `json:"username"` + Password string `json:"password"` + Host string `json:"host"` + Port json.RawMessage `json:"port"` + } + + if err := json.Unmarshal([]byte(secret), &raw); err != nil { + return fmt.Errorf("%w: the secret is not a JSON object: %w", ErrSecret, err) + } + + if raw.Username == "" || raw.Host == "" { + return fmt.Errorf("%w: the secret needs username, password, host and port members", ErrSecret) + } + + port, err := secretPort(raw.Port) + if err != nil { + return err + } + + ep.User, ep.Password, ep.Server, ep.Port = raw.Username, raw.Password, raw.Host, port + + return nil +} + +func secretPort(raw json.RawMessage) (int, error) { + if len(raw) == 0 { + return 0, fmt.Errorf("%w: the secret needs a port member", ErrSecret) + } + + var s json.Number + if err := json.Unmarshal(raw, &s); err != nil { + return 0, fmt.Errorf("%w: port: %w", ErrSecret, err) + } + + n, err := strconv.Atoi(s.String()) + if err != nil { + return 0, fmt.Errorf("%w: port %q is not a number", ErrSecret, s.String()) + } + + return n, nil +} diff --git a/pkgs/dmsengine/sql_target.go b/pkgs/dmsengine/sql_target.go index 5d426d1db2..51a22f63ce 100644 --- a/pkgs/dmsengine/sql_target.go +++ b/pkgs/dmsengine/sql_target.go @@ -65,7 +65,13 @@ func (t *sqlTarget) ph(n int) string { } func (t *sqlTarget) columnType(tt TargetTable, c TargetColumn) string { - typ := TargetType(tt.SourceFamily, t.fam, c.Native) + var typ string + + if c.DMS.Name != "" { + typ = c.DMS.SQLType(t.fam) + } else { + typ = TargetType(tt.SourceFamily, t.fam, c.Native) + } if t.fam == FamilyMySQL && (typ == tLongtext || typ == tText) && slicesContain(tt.PK, c.Name) { return varcharKey @@ -81,7 +87,7 @@ func (t *sqlTarget) createSQL(tt TargetTable) string { for _, c := range tt.Columns { def := t.q(c.Name) + " " + t.columnType(tt, c) - if !c.Nullable || slicesContain(tt.PK, c.Name) { + if !c.Nullable || (slicesContain(tt.PK, c.Name) && !tt.KeyUnique) { def += " NOT NULL" } @@ -89,12 +95,26 @@ func (t *sqlTarget) createSQL(tt TargetTable) string { } if len(tt.PK) > 0 { - defs = append(defs, "PRIMARY KEY ("+t.quoteAll(tt.PK)+")") + defs = append(defs, t.keyDef(tt)) } return "CREATE TABLE IF NOT EXISTS " + t.qualified(tt) + " (" + strings.Join(defs, ", ") + ")" } +func (t *sqlTarget) keyDef(tt TargetTable) string { + kind := "PRIMARY KEY" + if tt.KeyUnique { + kind = "UNIQUE" + } + + def := kind + " (" + t.quoteAll(tt.PK) + ")" + if tt.KeyName != "" { + def = "CONSTRAINT " + t.q(tt.KeyName) + " " + def + } + + return def +} + func (t *sqlTarget) quoteAll(names []string) string { q := make([]string, len(names)) for i, n := range names { diff --git a/pkgs/dmsengine/targets_test.go b/pkgs/dmsengine/targets_test.go index 0e17e83597..0966f9664e 100644 --- a/pkgs/dmsengine/targets_test.go +++ b/pkgs/dmsengine/targets_test.go @@ -34,6 +34,35 @@ func (m *memObjects) PutObject(_ context.Context, bucket, key string, body []byt return nil } +func (m *memObjects) ListObjects(_ context.Context, bucket, prefix string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + var keys []string + + for k := range m.objs { + if rest, ok := strings.CutPrefix(k, bucket+"/"); ok && strings.HasPrefix(rest, prefix) { + keys = append(keys, rest) + } + } + + sort.Strings(keys) + + return keys, nil +} + +func (m *memObjects) GetObject(_ context.Context, bucket, key string) ([]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + + body, ok := m.objs[bucket+"/"+key] + if !ok { + return nil, errBoom + } + + return []byte(body), nil +} + func (m *memObjects) CheckBucket(_ context.Context, bucket string) error { if bucket == "missing" { return errBoom diff --git a/pkgs/dmsengine/task.go b/pkgs/dmsengine/task.go index 0686e10cc7..9bc7fd41a0 100644 --- a/pkgs/dmsengine/task.go +++ b/pkgs/dmsengine/task.go @@ -85,9 +85,12 @@ type TaskConfig struct { Mappings Mappings PingBackoff time.Duration SourceFamily Family + // TargetFamily gates target-specific rules such as add-before-image-columns; FamilyOther skips the check. + TargetFamily Family } type planned struct { + filter *TableFilter src Table target TargetTable stat TableStat @@ -212,12 +215,26 @@ func (t *Task) setup(ctx context.Context) error { state = tableBeforeLoad } - for _, st := range t.cfg.Mappings.Select(all) { + selected, err := t.cfg.Mappings.SelectFiltered(all) + if err != nil { + return err + } + + for _, sel := range selected { + st := sel.Table + + target, perr := t.cfg.Mappings.Plan(st, t.cfg.SourceFamily, t.cfg.Settings.TargetSchema) + if perr != nil { + return perr + } + + if target.HasBeforeImage && !beforeImageTarget(t.cfg.TargetFamily) { + return fmt.Errorf("%w: add-before-image-columns is supported for Kinesis and S3 targets only", ErrMapping) + } + pl := &planned{ - src: st, - target: t.cfg.Mappings.Plan(st, t.cfg.SourceFamily, t.cfg.Settings.TargetSchema), - active: true, - stat: TableStat{Schema: st.Schema, Name: st.Name, State: state}, + src: st, filter: sel.Filter, target: target, active: true, + stat: TableStat{Schema: st.Schema, Name: st.Name, State: state}, } t.plans[st.Key()] = pl t.order = append(t.order, pl) @@ -226,6 +243,10 @@ func (t *Task) setup(ctx context.Context) error { return nil } +func beforeImageTarget(f Family) bool { + return f == FamilyOther || f == FamilyS3 || f == FamilyKinesis +} + func (t *Task) fullLoad(ctx context.Context) error { t.pmu.Lock() t.prog.FullLoadStart = t.cfg.Now() @@ -294,14 +315,16 @@ func (t *Task) copyTable(ctx context.Context, pl *planned, mode PrepMode, reload return nil } - err := t.cfg.Source.Scan(ctx, pl.src, func(r Row) error { - batch = append(batch, pl.target.Project(r)) + add := func(r Row) error { + batch = append(batch, pl.target.ProjectWith(r, ProjectCtx{Full: true, Now: t.cfg.Now()})) if len(batch) >= cap(batch) { return flush() } return nil - }) + } + + err := t.scan(ctx, pl, add) if err == nil { err = flush() } @@ -320,6 +343,29 @@ func (t *Task) copyTable(ctx context.Context, pl *planned, mode PrepMode, reload return nil } +// FilteredScanner is implemented by sources that push a selection filter into the full-load query. +type FilteredScanner interface { + ScanWhere(ctx context.Context, t Table, f *TableFilter, fn func(Row) error) error +} + +func (t *Task) scan(ctx context.Context, pl *planned, fn func(Row) error) error { + if pl.filter == nil { + return t.cfg.Source.Scan(ctx, pl.src, fn) + } + + if fs, ok := t.cfg.Source.(FilteredScanner); ok { + return fs.ScanWhere(ctx, pl.src, pl.filter, fn) + } + + return t.cfg.Source.Scan(ctx, pl.src, func(r Row) error { + if !pl.filter.Match(r) { + return nil + } + + return fn(r) + }) +} + // Reload re-copies tables while change capture keeps running. func (t *Task) Reload(ctx context.Context, refs []TableRef) error { t.applyMu.Lock() @@ -455,8 +501,14 @@ func (t *Task) apply(ctx context.Context, c Change) error { return nil } + c, ok := filterChange(pl.filter, c) + if !ok { + return nil + } + + pc := ProjectCtx{Op: c.Op, Time: c.Time, Now: t.cfg.Now(), Position: c.Position, Before: c.Before} tc := c - tc.Before, tc.After = pl.target.Project(c.Before), pl.target.Project(c.After) + tc.Before, tc.After = pl.target.ProjectWith(c.Before, pc), pl.target.ProjectWith(c.After, pc) if err := t.cfg.Target.Apply(ctx, pl.target, tc); err != nil { if ctx.Err() != nil { @@ -482,6 +534,47 @@ func (t *Task) apply(ctx context.Context, c Change) error { return nil } +// filterChange applies a selection filter to a change: rows entering the filter are inserted, rows leaving it are +// deleted, and rows that never matched are dropped. +func filterChange(f *TableFilter, c Change) (Change, bool) { + if f == nil { + return c, true + } + + switch c.Op { + case OpInsert: + return c, f.Match(c.After) + case OpDelete: + return c, c.Before == nil || f.Match(c.Before) + case OpUpdate: + return filterUpdate(f, c) + } + + return c, true +} + +func filterUpdate(f *TableFilter, c Change) (Change, bool) { + switch { + case f.Match(c.After): + if c.Before != nil && !f.Match(c.Before) { + c.Op = OpInsert + } + + return c, true + case c.Before == nil || f.Match(c.Before): + c.Op = OpDelete + if c.Before == nil { + c.Before = c.After + } + + c.After = nil + + return c, true + } + + return c, false +} + func rowWidth(src Table, c Change) string { for _, r := range []Row{c.Before, c.After} { if r != nil && len(r) != len(src.Columns) { diff --git a/pkgs/dmsengine/task_rules_test.go b/pkgs/dmsengine/task_rules_test.go new file mode 100644 index 0000000000..ab704bad9d --- /dev/null +++ b/pkgs/dmsengine/task_rules_test.go @@ -0,0 +1,188 @@ +package dmsengine_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func intTable() *fakeSource { + items := dmsengine.Table{Schema: "app", Name: "items", PK: []string{"id"}, Columns: []dmsengine.Column{ + {Name: "id", Native: "int", Kind: dmsengine.KindInt}, + {Name: "name", Native: "varchar(20)", Kind: dmsengine.KindString, Nullable: true}, + }} + + return &fakeSource{ + tables: []dmsengine.Table{items}, + rows: map[string][]dmsengine.Row{ + "app.items": {{"1", "alpha"}, {"2", "beta"}, {"30", "gamma"}}, + }, + changes: make(chan dmsengine.Change, 16), + } +} + +const filteredWithColumn = `{"rules":[ + {"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app","table-name":"items"}, + "rule-action":"include","filters":[{"filter-type":"source","column-name":"id", + "filter-conditions":[{"filter-operator":"ste","value":"10"}]}]}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column","rule-target":"column", + "object-locator":{"schema-name":"app","table-name":"items"},"value":"label", + "expression":"upper($name)||'-'||$id","data-type":{"type":"string","length":30}}]}` + +func TestTaskFilterAndAddColumnOnLoadAndCDC(t *testing.T) { + t.Parallel() + + src, tgt, log := intTable(), newFakeTarget(), newProgressLog() + mappings, err := dmsengine.ParseMappings(filteredWithColumn) + require.NoError(t, err) + + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: src, Target: tgt, MigrationType: dmsengine.MigrationFullLoadAndCDC, Mappings: mappings, + Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, OnProgress: log.push, + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan dmsengine.Result, 1) + + go func() { done <- task.Run(ctx) }() + + ins := func(id, name string) dmsengine.Change { + return dmsengine.Change{Schema: "app", Table: "items", Op: dmsengine.OpInsert, After: dmsengine.Row{id, name}} + } + upd := func(bid, bname, aid, aname string) dmsengine.Change { + return dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpUpdate, + Before: dmsengine.Row{bid, bname}, After: dmsengine.Row{aid, aname}, + } + } + + src.changes <- ins("5", "five") + src.changes <- ins("50", "fifty") + src.changes <- upd("5", "five", "5", "FIVE") + src.changes <- upd("5", "five", "99", "moved-out") + src.changes <- upd("60", "x", "3", "moved-in") + src.changes <- upd("70", "x", "71", "never") + src.changes <- dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpDelete, Before: dmsengine.Row{"80", "x"}, + } + src.changes <- dmsengine.Change{Commit: true, Position: "p"} + log.waitFor(t, func(p dmsengine.Progress) bool { return p.Checkpoint == "p" }) + cancel() + <-done + + assert.Equal(t, []dmsengine.Row{{"1", "alpha", "ALPHA-1"}, {"2", "beta", "BETA-2"}}, tgt.loaded["app.items"]) + + type applied struct { + op dmsengine.Op + row dmsengine.Row + } + + got := make([]applied, 0, len(tgt.applied)) + + for _, c := range tgt.applied { + img := c.After + if c.Op == dmsengine.OpDelete { + img = c.Before + } + + got = append(got, applied{c.Op, img}) + } + + assert.Equal(t, []applied{ + {dmsengine.OpInsert, dmsengine.Row{"5", "five", "FIVE-5"}}, + {dmsengine.OpUpdate, dmsengine.Row{"5", "FIVE", "FIVE-5"}}, + {dmsengine.OpDelete, dmsengine.Row{"5", "five", "FIVE-5"}}, + {dmsengine.OpInsert, dmsengine.Row{"3", "moved-in", "MOVED-IN-3"}}, + }, got) +} + +func TestTaskRuleErrorsFailTheTask(t *testing.T) { + t.Parallel() + + addCol := func(expr string) string { + return `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1",` + + `"object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-column","rule-target":"column", + "object-locator":{"schema-name":"app","table-name":"items"},"value":"x","expression":"` + expr + `", + "data-type":{"type":"string"}}]}` + } + + tests := []struct { + name string + doc string + want string + target dmsengine.Family + }{ + {name: "unsupported function", doc: addCol("soundex($name)"), target: dmsengine.FamilyOther, want: "soundex"}, + {name: "filter column missing", doc: `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1",` + + `"object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include",` + + `"filters":[{"filter-type":"source","column-name":"nope",` + + `"filter-conditions":[{"filter-operator":"null"}]}]}]}`, target: dmsengine.FamilyOther, want: "nope"}, + {name: "before image on a SQL target", doc: `{"rules":[{"rule-type":"selection","rule-id":"1",` + + `"rule-name":"1","object-locator":{"schema-name":"app","table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-before-image-columns", + "rule-target":"column","object-locator":{"schema-name":"app","table-name":"items"}}]}`, + target: dmsengine.FamilyPostgres, want: "Kinesis and S3"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + mappings, err := dmsengine.ParseMappings(tt.doc) + require.NoError(t, err) + + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: intTable(), Target: newFakeTarget(), MigrationType: dmsengine.MigrationFullLoad, + Mappings: mappings, Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, + TargetFamily: tt.target, + }) + + res := task.Run(t.Context()) + require.ErrorIs(t, res.Err, dmsengine.ErrMapping) + assert.Equal(t, dmsengine.StopFatalError, res.Reason) + assert.Contains(t, res.Err.Error(), tt.want) + }) + } +} + +func TestTaskBeforeImageOnKinesisLikeTarget(t *testing.T) { + t.Parallel() + + mappings, err := dmsengine.ParseMappings( + `{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"app",` + + `"table-name":"items"},"rule-action":"include"}, + {"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"add-before-image-columns", + "rule-target":"column","object-locator":{"schema-name":"app","table-name":"items"}}]}`, + ) + require.NoError(t, err) + + src, tgt, log := intTable(), newFakeTarget(), newProgressLog() + task := dmsengine.NewTask(dmsengine.TaskConfig{ + Source: src, Target: tgt, MigrationType: dmsengine.MigrationFullLoadAndCDC, Mappings: mappings, + Settings: defaultSettings(t), SourceFamily: dmsengine.FamilyMySQL, TargetFamily: dmsengine.FamilyKinesis, + OnProgress: log.push, + }) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan dmsengine.Result, 1) + + go func() { done <- task.Run(ctx) }() + + src.changes <- dmsengine.Change{ + Schema: "app", Table: "items", Op: dmsengine.OpUpdate, + Before: dmsengine.Row{"1", "alpha"}, After: dmsengine.Row{"1", "ALPHA"}, + } + src.changes <- dmsengine.Change{Commit: true, Position: "p"} + log.waitFor(t, func(p dmsengine.Progress) bool { return p.Checkpoint == "p" }) + cancel() + require.NoError(t, (<-done).Err) + + assert.Equal(t, dmsengine.Row{"1", "alpha", nil}, tgt.loaded["app.items"][0]) + require.Len(t, tgt.applied, 1) + assert.Equal(t, dmsengine.Row{"1", "ALPHA", "1"}, tgt.applied[0].After) +} diff --git a/pkgs/dmsengine/transform_test.go b/pkgs/dmsengine/transform_test.go new file mode 100644 index 0000000000..c2dfe2f9c3 --- /dev/null +++ b/pkgs/dmsengine/transform_test.go @@ -0,0 +1,293 @@ +package dmsengine_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/dmsengine" +) + +func transformDoc(rules ...string) string { + sel := `{"rule-type":"selection","rule-id":"1","rule-name":"1", + "object-locator":{"schema-name":"test","table-name":"employee"},"rule-action":"include"}` + + return `{"rules":[` + strings.Join(append([]string{sel}, rules...), ",") + `]}` +} + +func addColumn(id, name, expr, dataType string) string { + return `{"rule-type":"transformation","rule-id":"` + id + `","rule-name":"` + id + `","rule-action":"add-column", + "rule-target":"column","object-locator":{"schema-name":"test","table-name":"employee"},"value":"` + name + `", + "expression":"` + expr + `","data-type":` + dataType + `}` +} + +func TestAddColumnExpressions(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + row := dmsengine.Row{"7", "Tech", "20500.50", "2002-01-15", nil} + + tests := []struct { + want any + name string + expr string + typ string + }{ + {name: "concat", expr: `$dept||'_'||$empid`, typ: `{"type":"string","length":50}`, want: "Tech_7"}, + { + name: "case round", + expr: `CASE WHEN round($salary)>=20000 THEN 'SENIOR' ELSE 'JUNIOR' END`, + typ: `{"type":"string","length":50}`, + want: "SENIOR", + }, + {name: "round int8", expr: `round($salary)`, typ: `{"type":"int8"}`, want: "20501"}, + {name: "arithmetic", expr: `$empid*2+1`, typ: `{"type":"int4"}`, want: "15"}, + {name: "lower upper", expr: `lower($dept)||upper($dept)`, typ: `{"type":"string"}`, want: "techTECH"}, + {name: "substr hex", expr: `substr(hex($dept),1,4)`, typ: `{"type":"string"}`, want: "5465"}, + {name: "sha256", expr: `hash_sha256($dept)`, typ: `{"type":"string","length":64}`, want: ""}, + {name: "coalesce null", expr: `coalesce($notes,'none')`, typ: `{"type":"string"}`, want: "none"}, + {name: "datetime now", expr: `datetime()`, typ: `{"type":"datetime",` + + `"precision":6}`, want: "2026-03-04 05:06:07"}, + {name: "strftime", expr: `strftime('%Y/%m/%d', $started)`, typ: `{"type":"string"}`, want: "2002/01/15"}, + {name: "header operation", expr: `$AR_H_OPERATION`, typ: `{"type":"string"}`, want: "INSERT"}, + {name: "operation indicator", expr: `operation_indicator('D','U','I')`, typ: `{"type":"string"}`, want: "I"}, + { + name: "source metadata", + expr: `$AR_M_SOURCE_SCHEMA||'.'||$AR_M_SOURCE_TABLE_NAME`, + typ: `{"type":"string"}`, + want: "test.employee", + }, + {name: "boolean", expr: `$empid > 5`, typ: `{"type":"boolean"}`, want: "true"}, + {name: "replace trim", expr: `trim(replace($dept,'e',' '))`, typ: `{"type":"string"}`, want: "T ch"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(addColumn("2", "added", tt.expr, tt.typ))) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + out := plan.ProjectWith(row, dmsengine.ProjectCtx{Full: true, Now: now}) + require.Len(t, out, 6) + + if tt.name == "sha256" { + assert.Len(t, out[5], 64) + + return + } + + assert.Equal(t, tt.want, out[5]) + }) + } +} + +func TestAddColumnRejectsUnsupported(t *testing.T) { + t.Parallel() + + tests := []struct{ name, expr string }{ + {"unknown function", `soundex($dept)`}, + {"aggregate", `count($dept)`}, + {"missing column", `$nope`}, + {"change seq header", `$AR_H_CHANGE_SEQ`}, + {"syntax", `$dept ||`}, + {"modifiers", `date('now','+1 day')`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(addColumn("2", "added", tt.expr, `{"type":"string"}`))) + require.NoError(t, err) + + _, err = m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.ErrorIs(t, err, dmsengine.ErrMapping) + assert.Contains(t, err.Error(), "added") + }) + } +} + +func TestChangeDataTypeAndKeys(t *testing.T) { + t.Parallel() + + tests := []struct { + check func(*testing.T, dmsengine.TargetTable) + name string + rule string + wantCols []string + wantPK []string + }{ + { + name: "change type", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"empid"}, + "data-type":{"type":"string","length":"12"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, dmsengine.KindString, tt.Columns[0].Kind) + assert.Equal(t, "varchar(12)", tt.Columns[0].DMS.SQLType(dmsengine.FamilyPostgres)) + assert.Equal(t, "string", tt.Columns[0].Native) + }, + }, + { + name: "change type by existing data type", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"change-data-type",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"%", + "data-type":"numeric"},"data-type":{"type":"real8"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, dmsengine.KindFloat, tt.Columns[2].Kind) + assert.Equal(t, dmsengine.KindInt, tt.Columns[0].Kind) + }, + }, + { + name: "include-column", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2","rule-action":"include-column",` + + `"rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee","column-name":"dept"}}`, + wantCols: []string{"dept"}, wantPK: []string{}, + }, + { + name: "define-primary-key", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"define-primary-key","rule-target":"table", + "object-locator":{"schema-name":"test","table-name":"employee"},"primary-key-def":{"name":"emp_dept", + "origin":"unique-index","columns":["dept","started"]}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes"}, wantPK: []string{"dept", "started"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.Equal(t, "emp_dept", tt.KeyName) + assert.True(t, tt.KeyUnique) + }, + }, + { + name: "before image pk-only", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}, + "before-image-def":{"column-prefix":"BI_","column-suffix":"","column-filter":"pk-only"}}`, + wantCols: []string{"empid", "dept", "salary", "started", "notes", "BI_empid"}, wantPK: []string{"empid"}, + check: func(t *testing.T, tt dmsengine.TargetTable) { + t.Helper() + assert.True(t, tt.HasBeforeImage) + }, + }, + { + name: "before image non-lob defaults", + rule: `{"rule-type":"transformation","rule-id":"2","rule-name":"2",` + + `"rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}, + "before-image-def":{"column-suffix":"_old","column-prefix":"","column-filter":"non-lob"}}`, + wantCols: []string{ + "empid", + "dept", + "salary", + "started", + "notes", + "empid_old", + "dept_old", + "salary_old", + "started_old", + }, + wantPK: []string{"empid"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(tt.rule)) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + names := []string{} + for _, c := range plan.Columns { + names = append(names, c.Name) + } + + assert.Equal(t, tt.wantCols, names) + assert.Equal(t, tt.wantPK, plan.PK) + + if tt.check != nil { + tt.check(t, plan) + } + }) + } +} + +func TestBeforeImageProjection(t *testing.T) { + t.Parallel() + + m, err := dmsengine.ParseMappings(transformDoc(`{"rule-type":"transformation","rule-id":"2","rule-name":"2", + "rule-action":"add-before-image-columns","rule-target":"column", + "object-locator":{"schema-name":"test","table-name":"employee"}}`)) + require.NoError(t, err) + + plan, err := m.Plan(employeeTable(), dmsengine.FamilyMySQL, "") + require.NoError(t, err) + + before := dmsengine.Row{"1", "a", "1", "2020-01-01", nil} + after := dmsengine.Row{"3", "a", "1", "2020-01-01", nil} + + assert.Equal(t, dmsengine.Row{"3", "a", "1", "2020-01-01", nil, "1"}, + plan.ProjectWith(after, dmsengine.ProjectCtx{Op: dmsengine.OpUpdate, Before: before})) + assert.Equal(t, dmsengine.Row{"3", "a", "1", "2020-01-01", nil, nil}, plan.Project(after)) +} + +func TestDMSTypeSQL(t *testing.T) { + t.Parallel() + + tests := []struct { + pg string + my string + wantKindName string + dt dmsengine.DMSType + }{ + {dt: dmsengine.DMSType{Name: "int8"}, pg: "bigint", my: "bigint", wantKindName: "int8"}, + {dt: dmsengine.DMSType{Name: "uint4"}, pg: "bigint", my: "int unsigned", wantKindName: "uint4"}, + { + dt: dmsengine.DMSType{Name: "string", Length: 30}, + pg: "varchar(30)", + my: "varchar(30)", + wantKindName: "string", + }, + {dt: dmsengine.DMSType{Name: "string"}, pg: "text", my: "longtext", wantKindName: "string"}, + { + dt: dmsengine.DMSType{Name: "numeric", Precision: 5, Scale: 2}, + pg: "numeric(5,2)", + my: "numeric(5,2)", + wantKindName: "numeric", + }, + { + dt: dmsengine.DMSType{Name: "datetime", Scale: 3}, + pg: "timestamp(3)", + my: "datetime(3)", + wantKindName: "datetime", + }, + {dt: dmsengine.DMSType{Name: "boolean"}, pg: "boolean", my: "tinyint(1)", wantKindName: "boolean"}, + {dt: dmsengine.DMSType{Name: "bytes", Length: 16}, pg: "bytea", my: "varbinary(16)", wantKindName: "bytes"}, + } + + for _, tt := range tests { + t.Run(tt.wantKindName, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.pg, tt.dt.SQLType(dmsengine.FamilyPostgres)) + assert.Equal(t, tt.my, tt.dt.SQLType(dmsengine.FamilyMySQL)) + }) + } +} diff --git a/pkgs/dmsengine/typemap.go b/pkgs/dmsengine/typemap.go index 24c879ca9f..5cad051043 100644 --- a/pkgs/dmsengine/typemap.go +++ b/pkgs/dmsengine/typemap.go @@ -57,6 +57,26 @@ const ( tLongblob = "longblob" tJSONB = "jsonb" tDatetime6 = "datetime(6)" + tBlob = "blob" + tBinary = "binary" + tVarbinary = "varbinary" + tTinyblob = "tinyblob" + tMediumblob = "mediumblob" + tSerial = "serial" + tBigserial = "bigserial" + tInt1 = "int1" + tInt2 = "int2" + tInt4 = "int4" + tInt8 = "int8" + tReal4 = "real4" + tFloat4 = "float4" + tFloat8 = "float8" + tBool = "bool" + tClob = "clob" + tBytes = "bytes" + tReal8 = "real8" + tNclob = "nclob" + tTinyint1 = "tinyint(1)" ) func kindByBase() map[string]Kind { @@ -68,12 +88,12 @@ func kindByBase() map[string]Kind { tInteger: KindInt, tBigint: KindInt, "year": KindInt, - "serial": KindInt, - "bigserial": KindInt, + tSerial: KindInt, + tBigserial: KindInt, "smallserial": KindInt, - "int2": KindInt, - "int4": KindInt, - "int8": KindInt, + tInt2: KindInt, + tInt4: KindInt, + tInt8: KindInt, tDecimal: KindDecimal, tNumeric: KindDecimal, "dec": KindDecimal, @@ -83,10 +103,10 @@ func kindByBase() map[string]Kind { tDouble: KindFloat, tDoublePrec: KindFloat, tReal: KindFloat, - "float4": KindFloat, - "float8": KindFloat, + tFloat4: KindFloat, + tFloat8: KindFloat, tBoolean: KindBool, - "bool": KindBool, + tBool: KindBool, tDate: KindTime, tDatetime: KindTime, tTimestamp: KindTime, @@ -97,11 +117,11 @@ func kindByBase() map[string]Kind { "time with time zone": KindTime, "timestamptz": KindTime, "timetz": KindTime, - "binary": KindBytes, - "varbinary": KindBytes, - "tinyblob": KindBytes, - "blob": KindBytes, - "mediumblob": KindBytes, + tBinary: KindBytes, + tVarbinary: KindBytes, + tTinyblob: KindBytes, + tBlob: KindBytes, + tMediumblob: KindBytes, tLongblob: KindBytes, tBytea: KindBytes, "bit": KindBytes, @@ -120,7 +140,7 @@ func mysqlToPGBase() map[string]string { return map[string]string{ tTinyint: tSmallint, tMediumint: tInteger, "year": tSmallint, tFloat: tReal, tDouble: tDoublePrec, tDoublePrec: tDoublePrec, tDatetime: tTimestamp, tTimestamp: tTimestamp, tDate: tDate, tTime: tTime, - "binary": tBytea, "varbinary": tBytea, "tinyblob": tBytea, "blob": tBytea, "mediumblob": tBytea, + tBinary: tBytea, tVarbinary: tBytea, tTinyblob: tBytea, tBlob: tBytea, tMediumblob: tBytea, tLongblob: tBytea, "bit": tBytea, "geometry": tBytea, tJSON: tJSONB, } } @@ -172,10 +192,10 @@ func numericWithArgs(args string) string { func pgToMySQLBase() map[string]string { return map[string]string{ - tSmallint: tSmallint, "int2": tSmallint, "smallserial": tSmallint, tInteger: tInt, tInt: tInt, "int4": tInt, - "serial": tInt, tBigint: tBigint, "int8": tBigint, "bigserial": tBigint, tBoolean: "tinyint(1)", - "bool": "tinyint(1)", "money": "decimal(19,2)", tReal: tFloat, "float4": tFloat, tDoublePrec: tDouble, - "float8": tDouble, tTimestampN: tDatetime6, tTimestampZ: tDatetime6, tTimestamp: tDatetime6, + tSmallint: tSmallint, tInt2: tSmallint, "smallserial": tSmallint, tInteger: tInt, tInt: tInt, tInt4: tInt, + tSerial: tInt, tBigint: tBigint, tInt8: tBigint, tBigserial: tBigint, tBoolean: tTinyint1, + tBool: tTinyint1, "money": "decimal(19,2)", tReal: tFloat, tFloat4: tFloat, tDoublePrec: tDouble, + tFloat8: tDouble, tTimestampN: tDatetime6, tTimestampZ: tDatetime6, tTimestamp: tDatetime6, "timestamptz": tDatetime6, tDate: tDate, "time without time zone": tTime, "time with time zone": tTime, tTime: tTime, "timetz": tTime, tBytea: tLongblob, tJSON: tJSON, tJSONB: tJSON, "uuid": "char(36)", } diff --git a/pkgs/httpclient/httpclient.go b/pkgs/httpclient/httpclient.go index 7697196221..56b2208e51 100644 --- a/pkgs/httpclient/httpclient.go +++ b/pkgs/httpclient/httpclient.go @@ -3,6 +3,7 @@ package httpclient import ( "net/http" + "net/netip" "time" ) @@ -22,3 +23,11 @@ func NewTransport() *http.Transport { func New(timeout time.Duration) *http.Client { return &http.Client{Timeout: timeout, Transport: NewTransport()} } + +// IsLoopbackHost reports whether host is a loopback IP literal; the only targets where TLS verification may be +// skipped (emulator-run backends; policy 2026-10-31). +func IsLoopbackHost(host string) bool { + ip, err := netip.ParseAddr(host) + + return err == nil && ip.IsLoopback() +} diff --git a/pkgs/httpclient/httpclient_test.go b/pkgs/httpclient/httpclient_test.go index 98ad7a4112..a62ffa9557 100644 --- a/pkgs/httpclient/httpclient_test.go +++ b/pkgs/httpclient/httpclient_test.go @@ -82,3 +82,21 @@ func TestNewTransportDistinct(t *testing.T) { assert.NotSame(t, a, b) assert.NotNil(t, a.Proxy) } + +func TestIsLoopbackHost(t *testing.T) { + t.Parallel() + + tests := []struct { + host string + want bool + }{ + {"127.0.0.1", true}, {"::1", true}, {"localhost", false}, {"10.0.0.1", false}, {"", false}, + } + + for _, tt := range tests { + t.Run(tt.host, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, httpclient.IsLoopbackHost(tt.host)) + }) + } +} diff --git a/pkgs/k3sengine/image.go b/pkgs/k3sengine/image.go index 583f24f5b2..cd811cee27 100644 --- a/pkgs/k3sengine/image.go +++ b/pkgs/k3sengine/image.go @@ -38,7 +38,7 @@ const scriptHead = `printf '%s,admin,admin,"system:masters"\n' "$` + envAdminAut ` && { [ -s ` + tlsDir + `/server-ca.crt ] && [ -s ` + tlsDir + `/server-ca.key ] || {` + ` printf '%s\n' "$` + envCACert + `" > ` + tlsDir + `/server-ca.crt` + ` && printf '%s\n' "$` + envCAKey + `" > ` + tlsDir + `/server-ca.key; }; }` + - ` && unset ` + envCAKey + ` && exec /bin/k3s server --tls-san "$` + envHost + `"` + + ` && unset ` + envCAKey + ` && umask 022 && exec /bin/k3s server --tls-san "$` + envHost + `"` + ` --disable traefik${` + envNodeName + `:+ --node-name "$` + envNodeName + `"}` const scriptTail = ` --disable metrics-server --kube-apiserver-arg=token-auth-file=` + tokenFileName diff --git a/pkgs/k3sengine/k3sengine_test.go b/pkgs/k3sengine/k3sengine_test.go index cb238fd772..75ceb6047f 100644 --- a/pkgs/k3sengine/k3sengine_test.go +++ b/pkgs/k3sengine/k3sengine_test.go @@ -95,6 +95,13 @@ func TestScript(t *testing.T) { assert.Contains(t, k3sengine.Script(false), "--disable servicelb") assert.NotContains(t, k3sengine.Script(true), "servicelb") + // umask 077 for the CA key must not leak into k3s/containerd: it would make every image root 0700. + for _, lb := range []bool{false, true} { + sc := k3sengine.Script(lb) + assert.Less(t, strings.Index(sc, "umask 077"), strings.Index(sc, "umask 022")) + assert.Less(t, strings.Index(sc, "umask 022"), strings.Index(sc, "exec /bin/k3s")) + } + for _, lb := range []bool{false, true} { s := k3sengine.Script(lb) assert.Contains(t, s, "token-auth-file=") diff --git a/pkgs/persistence/testdata/snapshot_inventory.json b/pkgs/persistence/testdata/snapshot_inventory.json index 298ca89f5d..058fc0058d 100644 --- a/pkgs/persistence/testdata/snapshot_inventory.json +++ b/pkgs/persistence/testdata/snapshot_inventory.json @@ -6684,6 +6684,7 @@ "StackEvent.StackName string `xml:\"StackName\" json:\"stackName\"`", "StackEvent.Timestamp time.Time `xml:\"Timestamp\" json:\"timestamp\"`", "StackInstance.Account string `xml:\"Account,omitempty\" json:\"account,omitempty\"`", + "StackInstance.DetailedStatus string `xml:\"-\" json:\"detailedStatus,omitempty\"`", "StackInstance.DriftStatus string `xml:\"DriftStatus,omitempty\" json:\"driftStatus,omitempty\"`", "StackInstance.LastDriftCheckTimestamp *time.Time `xml:\"LastDriftCheckTimestamp,omitempty\" json:\"lastDriftCheckTimestamp,omitempty\"`", "StackInstance.LastOperationID string `xml:\"LastOperationId,omitempty\" json:\"lastOperationID,omitempty\"`", @@ -6774,13 +6775,18 @@ "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.TypeConfigs map[string]string `json:\"typeConfigs\"`", "backendSnapshot.TypeVersions map[string][]*RegisteredTypeVersion `json:\"typeVersions\"`", + "pendingStackOp.Announced []string `json:\"announced,omitempty\"`", + "pendingStackOp.Created []string `json:\"created,omitempty\"`", "pendingStackOp.DisableValidation bool `json:\"disableValidation,omitempty\"`", + "pendingStackOp.ForceDelete bool `json:\"forceDelete,omitempty\"`", "pendingStackOp.Kind string `json:\"kind\"`", "pendingStackOp.OldParams []Parameter `json:\"oldParams,omitempty\"`", "pendingStackOp.OldTemplateBody string `json:\"oldTemplateBody,omitempty\"`", "pendingStackOp.OnFailure string `json:\"onFailure,omitempty\"`", + "pendingStackOp.PrevResources map[string]*StackResource `json:\"prevResources,omitempty\"`", "pendingStackOp.ResourceTypes []string `json:\"resourceTypes,omitempty\"`", - "pendingStackOp.RetainExceptOnCreate bool `json:\"retainExceptOnCreate,omitempty\"`" + "pendingStackOp.RetainExceptOnCreate bool `json:\"retainExceptOnCreate,omitempty\"`", + "pendingStackOp.RetainResources []string `json:\"retainResources,omitempty\"`" ], "version": 1 }, @@ -10794,18 +10800,29 @@ "ReplicationConfig.CdcStartTime *time.Time `json:\",omitempty\"`", "ReplicationConfig.CdcStopPosition string `json:\",omitempty\"`", "ReplicationConfig.ComputeConfig *ComputeConfig", + "ReplicationConfig.CreateTime time.Time `json:\",omitzero\"`", + "ReplicationConfig.ElapsedTimeMillis int64 `json:\"-\"`", + "ReplicationConfig.FailureMessage string `json:\",omitempty\"`", + "ReplicationConfig.FullLoadFinishDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.FullLoadStartDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.RecoveryCheckpoint string `json:\",omitempty\"`", "ReplicationConfig.Region string", "ReplicationConfig.ReplicationConfigArn string", "ReplicationConfig.ReplicationConfigIdentifier string", "ReplicationConfig.ReplicationSettings string `json:\",omitempty\"`", "ReplicationConfig.ReplicationType string", "ReplicationConfig.SourceEndpointArn string", + "ReplicationConfig.StartDate time.Time `json:\",omitzero\"`", "ReplicationConfig.StartReplicationType string", "ReplicationConfig.Status string", + "ReplicationConfig.StopDate time.Time `json:\",omitzero\"`", + "ReplicationConfig.StopReason string `json:\",omitempty\"`", "ReplicationConfig.SupplementalSettings string `json:\",omitempty\"`", "ReplicationConfig.TableMappings string", + "ReplicationConfig.TableStats []TableStatistic `json:\",omitempty\"`", "ReplicationConfig.Tags *tags.Tags `json:\"-\"`", "ReplicationConfig.TargetEndpointArn string", + "ReplicationConfig.UpdateTime time.Time `json:\",omitzero\"`", "ReplicationInstance.AccountID string `json:\"accountId\"`", "ReplicationInstance.AllocatedStorage int32 `json:\"allocatedStorage\"`", "ReplicationInstance.AutoMinorVersionUpgrade bool `json:\"autoMinorVersionUpgrade\"`", @@ -16910,20 +16927,37 @@ "Bucket.Location string `json:\"location\"`", "Bucket.Metageneration int64 `json:\"metageneration\"`", "Bucket.Name string `json:\"name\"`", + "Bucket.NotifSeq int64 `json:\"notifSeq,omitempty\"`", + "Bucket.Notifications []Notification `json:\"notifications,omitempty\"`", "Bucket.Objects map[string][]*Object `json:\"objects\"`", "Bucket.Project string `json:\"project\"`", + "Bucket.SoftDeleted []*Object `json:\"softDeleted,omitempty\"`", "Bucket.StorageClass string `json:\"storageClass\"`", "Bucket.Updated time.Time `json:\"updated\"`", "Bucket.Versioning bool `json:\"versioning,omitempty\"`", + "HMACKey.AccessID string `json:\"accessId\"`", + "HMACKey.Created time.Time `json:\"created\"`", + "HMACKey.Project string `json:\"project\"`", + "HMACKey.Secret string `json:\"secret\"`", + "HMACKey.ServiceAccountEmail string `json:\"serviceAccountEmail\"`", + "HMACKey.State string `json:\"state\"`", + "HMACKey.Updated time.Time `json:\"updated\"`", "IAMBinding.Condition json.RawMessage `json:\"condition,omitempty\"`", "IAMBinding.Members []string `json:\"members\"`", "IAMBinding.Role string `json:\"role\"`", "IAMPolicy.Bindings []IAMBinding `json:\"bindings\"`", "IAMPolicy.Etag string `json:\"etag\"`", + "Notification.CustomAttributes map[string]string `json:\"customAttributes,omitempty\"`", + "Notification.EventTypes []string `json:\"eventTypes,omitempty\"`", + "Notification.ID string `json:\"id\"`", + "Notification.ObjectNamePrefix string `json:\"objectNamePrefix,omitempty\"`", + "Notification.PayloadFormat string `json:\"payloadFormat\"`", + "Notification.Topic string `json:\"topic\"`", "Object.ACL []ACLEntry `json:\"acl,omitempty\"`", "Object.Bucket string `json:\"bucket\"`", "Object.CRC32C string `json:\"crc32c\"`", "Object.CacheControl string `json:\"cacheControl,omitempty\"`", + "Object.ClassUpdated time.Time `json:\"classUpdated\"`", "Object.ComponentCount int64 `json:\"componentCount,omitempty\"`", "Object.ContentDisposition string `json:\"contentDisposition,omitempty\"`", "Object.ContentEncoding string `json:\"contentEncoding,omitempty\"`", @@ -16935,15 +16969,20 @@ "Object.Deleted time.Time `json:\"deleted\"`", "Object.EventBasedHold bool `json:\"eventBasedHold,omitempty\"`", "Object.Generation int64 `json:\"generation\"`", + "Object.HardDeleteTime time.Time `json:\"hardDeleteTime\"`", + "Object.HoldReleased time.Time `json:\"holdReleased\"`", "Object.MD5 string `json:\"md5,omitempty\"`", "Object.Metadata map[string]string `json:\"metadata,omitempty\"`", "Object.Metageneration int64 `json:\"metageneration\"`", "Object.Name string `json:\"name\"`", + "Object.SoftDeleteTime time.Time `json:\"softDeleteTime\"`", "Object.StorageClass string `json:\"storageClass\"`", "Object.TemporaryHold bool `json:\"temporaryHold,omitempty\"`", "Object.Updated time.Time `json:\"updated\"`", "Object.hideACL bool", - "backendSnapshot.Buckets map[string]*Bucket `json:\"buckets\"`" + "Object.retention time.Time", + "backendSnapshot.Buckets map[string]*Bucket `json:\"buckets\"`", + "backendSnapshot.HMACKeys map[string]*HMACKey `json:\"hmacKeys,omitempty\"`" ], "version": 1 }, @@ -21334,6 +21373,79 @@ ], "version": 1 }, + "lambdaweb": { + "fields": [ + "Endpoint.ARN string", + "Endpoint.AccountID string", + "Endpoint.AuthType string", + "Endpoint.AutoDeploymentMode string", + "Endpoint.CreatedAt time.Time", + "Endpoint.Description string", + "Endpoint.DomainName string", + "Endpoint.FunctionName string", + "Endpoint.MaxEnvironments *int32", + "Endpoint.Name string", + "Endpoint.PendingUntil time.Time", + "Endpoint.RateLimit *int32", + "Endpoint.Region string", + "Endpoint.RegionDomains map[string]string", + "Endpoint.Regions []string", + "Endpoint.State string", + "Endpoint.StateReason string", + "Endpoint.Type string", + "Endpoint.UpdatePendingUntil time.Time", + "Endpoint.UpdateReason string", + "Endpoint.UpdateStatus string", + "Endpoint.UpdatedAt time.Time", + "Endpoint.Weights []RevisionWeight", + "Function.ARN string", + "Function.AccountID string", + "Function.CreatedAt time.Time", + "Function.Name string", + "Function.PendingUntil time.Time", + "Function.Policy string", + "Function.PolicyRevID string", + "Function.Region string", + "Function.State string", + "Function.StateReason string", + "Function.Tags map[string]string", + "Function.UpdatedAt time.Time", + "LoggingConfig.ApplicationLogLevel string", + "LoggingConfig.LogGroup string", + "LoggingConfig.SystemLogLevel string", + "Revision.ARN string", + "Revision.AccountID string", + "Revision.CreatedAt time.Time", + "Revision.Description string", + "Revision.Errors []RevisionError", + "Revision.FunctionName string", + "Revision.ID string", + "Revision.KmsKeyArn string", + "Revision.PendingUntil time.Time", + "Revision.Region string", + "Revision.Runtime string", + "Revision.S3 S3Object", + "Revision.ServiceConfig ServiceConfig", + "Revision.State string", + "Revision.StateReason string", + "RevisionError.Attribute string", + "RevisionError.ErrorCode string", + "RevisionError.ErrorMessage string", + "RevisionWeight.RevisionID string", + "RevisionWeight.Weight int32", + "S3Object.Bucket string", + "S3Object.Key string", + "S3Object.VersionID string", + "ServiceConfig.EgressNetworkConnectorArn string", + "ServiceConfig.EnvironmentVariables map[string]string", + "ServiceConfig.ExecutionRoleArn string", + "ServiceConfig.Logging *LoggingConfig", + "ServiceConfig.MaxConcurrencyPerEnvironment int32", + "ServiceConfig.TimeoutSeconds int32", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`" + ], + "version": 1 + }, "lightsail": { "fields": [ "AccessKey.AccessKeyID string", @@ -25840,6 +25952,180 @@ ], "version": 1 }, + "ocidatabase": { + "fields": [ + "ADB.AccessControl bool `json:\"accessControl\"`", + "ADB.AutoScaling bool `json:\"autoScaling\"`", + "ADB.BackupDays int `json:\"backupRetentionDays\"`", + "ADB.CharacterSet string `json:\"characterSet\"`", + "ADB.CompartmentID string `json:\"compartmentId\"`", + "ADB.ComputeCount float32 `json:\"computeCount\"`", + "ADB.ComputeModel string `json:\"computeModel\"`", + "ADB.Created time.Time `json:\"created\"`", + "ADB.DBName string `json:\"dbName\"`", + "ADB.DBVersion string `json:\"dbVersion\"`", + "ADB.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", + "ADB.Details string `json:\"details,omitempty\"`", + "ADB.DisplayName string `json:\"displayName\"`", + "ADB.EndpointIP string `json:\"privateEndpointIp,omitempty\"`", + "ADB.EndpointLabel string `json:\"privateEndpointLabel,omitempty\"`", + "ADB.Engine bool `json:\"engine,omitempty\"`", + "ADB.Etag string `json:\"etag\"`", + "ADB.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "ADB.ID string `json:\"id\"`", + "ADB.IsFreeTier bool `json:\"isFreeTier\"`", + "ADB.LicenseModel string `json:\"licenseModel\"`", + "ADB.Mtls bool `json:\"mtlsRequired\"`", + "ADB.NCharacterSet string `json:\"ncharacterSet\"`", + "ADB.NsgIDs []string `json:\"nsgIds,omitempty\"`", + "ADB.Seq int64 `json:\"seq\"`", + "ADB.State string `json:\"state\"`", + "ADB.StorageGBs int `json:\"storageGBs\"`", + "ADB.StorageScales bool `json:\"storageAutoScaling\"`", + "ADB.SubnetID string `json:\"subnetId,omitempty\"`", + "ADB.Target string `json:\"target,omitempty\"`", + "ADB.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "ADB.UniqueID string `json:\"uniqueId\"`", + "ADB.WalletRotated *time.Time `json:\"walletRotated,omitempty\"`", + "ADB.Whitelisted []string `json:\"whitelistedIps,omitempty\"`", + "ADB.Workload string `json:\"workload\"`", + "Backup.AvailableTill *time.Time `json:\"availableTill,omitempty\"`", + "Backup.CompartmentID string `json:\"compartmentId\"`", + "Backup.DBID string `json:\"autonomousDatabaseId\"`", + "Backup.DBVersion string `json:\"dbVersion\"`", + "Backup.Details string `json:\"details,omitempty\"`", + "Backup.DisplayName string `json:\"displayName\"`", + "Backup.Ended *time.Time `json:\"ended,omitempty\"`", + "Backup.ID string `json:\"id\"`", + "Backup.IsAutomatic bool `json:\"isAutomatic\"`", + "Backup.RetentionDays int `json:\"retentionDays\"`", + "Backup.Seq int64 `json:\"seq\"`", + "Backup.SizeTBs float64 `json:\"sizeInTBs\"`", + "Backup.Started time.Time `json:\"started\"`", + "Backup.State string `json:\"state\"`", + "Backup.TransitionAt *time.Time `json:\"transitionAt,omitempty\"`", + "Backup.Type string `json:\"type\"`", + "WorkRequest.Accepted time.Time `json:\"accepted\"`", + "WorkRequest.Action string `json:\"action\"`", + "WorkRequest.Compartment string `json:\"compartmentId\"`", + "WorkRequest.ErrCode string `json:\"errCode,omitempty\"`", + "WorkRequest.ErrMsg string `json:\"errMsg,omitempty\"`", + "WorkRequest.FinishAt *time.Time `json:\"finishAt,omitempty\"`", + "WorkRequest.FinishedAt *time.Time `json:\"finishedAt,omitempty\"`", + "WorkRequest.ID string `json:\"id\"`", + "WorkRequest.ResourceID string `json:\"resourceId\"`", + "WorkRequest.ResourceType string `json:\"resourceType\"`", + "WorkRequest.Seq int64 `json:\"seq\"`", + "WorkRequest.Status string `json:\"status\"`", + "WorkRequest.Type string `json:\"type\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Backups map[string]*Backup `json:\"backups\"`", + "backendSnapshot.Databases map[string]*ADB `json:\"databases\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.Tokens map[string]string `json:\"retryTokens\"`", + "backendSnapshot.WorkReqs map[string]*WorkRequest `json:\"workRequests\"`" + ], + "version": 1 + }, + "ocidns": { + "fields": [ + "Attachment.CompartmentID string `json:\"compartmentId\"`", + "Attachment.Created time.Time `json:\"created\"`", + "Attachment.DisplayName string `json:\"displayName\"`", + "Attachment.Domain string `json:\"domain\"`", + "Attachment.ID string `json:\"id\"`", + "Attachment.Rtypes []string `json:\"rtypes\"`", + "Attachment.Seq int64 `json:\"seq\"`", + "Attachment.SteeringPolicyID string `json:\"steeringPolicyId\"`", + "Attachment.ZoneID string `json:\"zoneId\"`", + "Endpoint.CompartmentID string `json:\"compartmentId\"`", + "Endpoint.Created time.Time `json:\"created\"`", + "Endpoint.ForwardingAddress string `json:\"forwardingAddress,omitempty\"`", + "Endpoint.ID string `json:\"id\"`", + "Endpoint.IsForwarding bool `json:\"isForwarding\"`", + "Endpoint.IsListening bool `json:\"isListening\"`", + "Endpoint.ListeningAddress string `json:\"listeningAddress,omitempty\"`", + "Endpoint.Name string `json:\"name\"`", + "Endpoint.NsgIDs []string `json:\"nsgIds\"`", + "Endpoint.ResolverID string `json:\"resolverId\"`", + "Endpoint.Seq int64 `json:\"seq\"`", + "Endpoint.SubnetID string `json:\"subnetId\"`", + "Endpoint.Updated time.Time `json:\"updated\"`", + "ExtServer.Address string `json:\"address\"`", + "ExtServer.Port *int `json:\"port,omitempty\"`", + "ExtServer.TsigKeyID string `json:\"tsigKeyId,omitempty\"`", + "RRSet.Domain string `json:\"domain\"`", + "RRSet.Rdata []string `json:\"rdata\"`", + "RRSet.Rtype string `json:\"rtype\"`", + "RRSet.TTL int `json:\"ttl\"`", + "Resolver.AttachedViews []string `json:\"attachedViews\"`", + "Resolver.CompartmentID string `json:\"compartmentId\"`", + "Resolver.Created time.Time `json:\"created\"`", + "Resolver.DefaultViewID string `json:\"defaultViewId\"`", + "Resolver.DisplayName string `json:\"displayName\"`", + "Resolver.ID string `json:\"id\"`", + "Resolver.Rules []ResolverRule `json:\"rules\"`", + "Resolver.Seq int64 `json:\"seq\"`", + "Resolver.Updated time.Time `json:\"updated\"`", + "Resolver.VcnID string `json:\"vcnId\"`", + "ResolverRule.ClientAddressConditions []string `json:\"clientAddressConditions\"`", + "ResolverRule.DestinationAddresses []string `json:\"destinationAddresses\"`", + "ResolverRule.QnameCoverConditions []string `json:\"qnameCoverConditions\"`", + "ResolverRule.SourceEndpointName string `json:\"sourceEndpointName\"`", + "SteeringPolicy.Answers []map[string]any `json:\"answers\"`", + "SteeringPolicy.CompartmentID string `json:\"compartmentId\"`", + "SteeringPolicy.Created time.Time `json:\"created\"`", + "SteeringPolicy.DisplayName string `json:\"displayName\"`", + "SteeringPolicy.HealthCheckMonitorID string `json:\"healthCheckMonitorId,omitempty\"`", + "SteeringPolicy.ID string `json:\"id\"`", + "SteeringPolicy.Rules []map[string]any `json:\"rules\"`", + "SteeringPolicy.Seq int64 `json:\"seq\"`", + "SteeringPolicy.TTL int `json:\"ttl\"`", + "SteeringPolicy.Template string `json:\"template\"`", + "Tags.DefinedTags map[string]map[string]any `json:\"definedTags,omitempty\"`", + "Tags.FreeformTags map[string]string `json:\"freeformTags,omitempty\"`", + "TsigKey.Algorithm string `json:\"algorithm\"`", + "TsigKey.CompartmentID string `json:\"compartmentId\"`", + "TsigKey.Created time.Time `json:\"created\"`", + "TsigKey.ID string `json:\"id\"`", + "TsigKey.Name string `json:\"name\"`", + "TsigKey.Secret string `json:\"secret\"`", + "TsigKey.Seq int64 `json:\"seq\"`", + "TsigKey.Updated time.Time `json:\"updated\"`", + "View.CompartmentID string `json:\"compartmentId\"`", + "View.Created time.Time `json:\"created\"`", + "View.DisplayName string `json:\"displayName\"`", + "View.ID string `json:\"id\"`", + "View.Protected bool `json:\"protected\"`", + "View.Seq int64 `json:\"seq\"`", + "View.Updated time.Time `json:\"updated\"`", + "Zone.CompartmentID string `json:\"compartmentId\"`", + "Zone.Created time.Time `json:\"created\"`", + "Zone.DnssecState string `json:\"dnssecState\"`", + "Zone.ExternalDownstreams []ExtServer `json:\"externalDownstreams\"`", + "Zone.ExternalMasters []ExtServer `json:\"externalMasters\"`", + "Zone.ID string `json:\"id\"`", + "Zone.Name string `json:\"name\"`", + "Zone.Records map[string]*RRSet `json:\"records\"`", + "Zone.ResolutionMode string `json:\"resolutionMode\"`", + "Zone.Scope string `json:\"scope\"`", + "Zone.Seq int64 `json:\"seq\"`", + "Zone.ViewID string `json:\"viewId,omitempty\"`", + "Zone.ZoneType string `json:\"zoneType\"`", + "backendSnapshot.Attachments map[string]*Attachment `json:\"steeringPolicyAttachments\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Endpoints map[string]*Endpoint `json:\"endpoints\"`", + "backendSnapshot.Policies map[string]*SteeringPolicy `json:\"steeringPolicies\"`", + "backendSnapshot.Resolvers map[string]*Resolver `json:\"resolvers\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.TsigKeys map[string]*TsigKey `json:\"tsigKeys\"`", + "backendSnapshot.Views map[string]*View `json:\"views\"`", + "backendSnapshot.Zones map[string]*Zone `json:\"zones\"`", + "meta.ETagSeq int64 `json:\"etagSeq\"`", + "meta.Modified time.Time `json:\"modified\"`" + ], + "version": 1 + }, "ocievents": { "fields": [ "Action.ActionType string `json:\"actionType\"`", @@ -25871,6 +26157,50 @@ ], "version": 1 }, + "ocifilestorage": { + "fields": [ + "Snapshot.ExclusiveBytes int64 `json:\"exclusiveBytes\"`", + "Snapshot.ExpirationTime *time.Time `json:\"expirationTime,omitempty\"`", + "Snapshot.FileSystemID string `json:\"fileSystemId\"`", + "Snapshot.FilesystemSnapshotPolicyID string `json:\"filesystemSnapshotPolicyId,omitempty\"`", + "Snapshot.IsCloneSource bool `json:\"isCloneSource\"`", + "Snapshot.LifecycleDetails string `json:\"lifecycleDetails\"`", + "Snapshot.LockDurationDetails *lockDuration `json:\"lockDurationDetails,omitempty\"`", + "Snapshot.Name string `json:\"name\"`", + "Snapshot.ProvenanceID string `json:\"provenanceId\"`", + "Snapshot.SnapshotTime time.Time `json:\"snapshotTime\"`", + "Snapshot.SnapshotType string `json:\"snapshotType\"`", + "Snapshot.TimeLocked *time.Time `json:\"timeLocked,omitempty\"`", + "backendSnapshot.Auth []ociauth.KeyRecord `json:\"authKeys\"`", + "backendSnapshot.Connectors table[*Connector] `json:\"connectors\"`", + "backendSnapshot.ExportSets table[*ExportSet] `json:\"exportSets\"`", + "backendSnapshot.Exports table[*Export] `json:\"exports\"`", + "backendSnapshot.FileSystems table[*FileSystem] `json:\"fileSystems\"`", + "backendSnapshot.MountTargets table[*MountTarget] `json:\"mountTargets\"`", + "backendSnapshot.Policies table[*Policy] `json:\"policies\"`", + "backendSnapshot.Quotas table[*QuotaRule] `json:\"quotaRules\"`", + "backendSnapshot.Replications table[*Replication] `json:\"replications\"`", + "backendSnapshot.Seq int64 `json:\"seq\"`", + "backendSnapshot.Snapshots table[*Snapshot] `json:\"snapshots\"`", + "backendSnapshot.Targets table[*ReplicationTarget] `json:\"replicationTargets\"`", + "backendSnapshot.Tokens map[string]string `json:\"retryTokens\"`", + "base.DefinedTags map[string]map[string]any `json:\"definedTags\"`", + "base.FreeformTags map[string]string `json:\"freeformTags\"`", + "base.ID string `json:\"id\"`", + "base.Locks []lock `json:\"locks\"`", + "base.State string `json:\"lifecycleState\"`", + "base.SystemTags map[string]map[string]any `json:\"systemTags,omitempty\"`", + "base.TimeCreated time.Time `json:\"timeCreated\"`", + "lock.Message string `json:\"message,omitempty\"`", + "lock.RelatedResourceID string `json:\"relatedResourceId,omitempty\"`", + "lock.TimeCreated *time.Time `json:\"timeCreated,omitempty\"`", + "lock.Type string `json:\"type\"`", + "lockDuration.CoolOffDuration *int `json:\"coolOffDuration,omitempty\"`", + "lockDuration.LockDuration *int `json:\"lockDuration,omitempty\"`", + "lockDuration.LockMode string `json:\"lockMode\"`" + ], + "version": 1 + }, "ocifunctions": { "fields": [ "Application.CompartmentID string `json:\"compartmentId\"`", @@ -35956,14 +36286,18 @@ "CallAnalyticsJob.FailureReason string `json:\"failureReason,omitempty\"`", "CallAnalyticsJob.IdentifiedLanguageScore float32 `json:\"identifiedLanguageScore,omitempty\"`", "CallAnalyticsJob.LanguageCode string `json:\"languageCode\"`", + "CallAnalyticsJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "CallAnalyticsJob.ManagedToken string `json:\"managedToken,omitempty\"`", "CallAnalyticsJob.Media Media `json:\"media\"`", "CallAnalyticsJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "CallAnalyticsJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", "CallAnalyticsJob.OutputLocation string `json:\"outputLocation,omitempty\"`", + "CallAnalyticsJob.RedactedTranscriptURI string `json:\"redactedTranscriptFileUri,omitempty\"`", "CallAnalyticsJob.Settings *CallAnalyticsSettings `json:\"settings,omitempty\"`", "CallAnalyticsJob.StartTime time.Time `json:\"startTime\"`", "CallAnalyticsJob.Tags map[string]string `json:\"tags,omitempty\"`", - "CallAnalyticsJob.TranscriptJSON []byte `json:\"-\"`", + "CallAnalyticsJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "CallAnalyticsJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "CallAnalyticsRule.InterruptionFilter *InterruptionFilter `json:\"InterruptionFilter,omitempty\"`", "CallAnalyticsRule.NonTalkTimeFilter *NonTalkTimeFilter `json:\"NonTalkTimeFilter,omitempty\"`", "CallAnalyticsRule.SentimentFilter *SentimentFilter `json:\"SentimentFilter,omitempty\"`", @@ -36039,6 +36373,8 @@ "MedicalTranscriptionJob.CreationTime time.Time `json:\"creationTime\"`", "MedicalTranscriptionJob.FailureReason string `json:\"failureReason,omitempty\"`", "MedicalTranscriptionJob.LanguageCode string `json:\"languageCode\"`", + "MedicalTranscriptionJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "MedicalTranscriptionJob.ManagedToken string `json:\"managedToken,omitempty\"`", "MedicalTranscriptionJob.Media Media `json:\"media\"`", "MedicalTranscriptionJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "MedicalTranscriptionJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", @@ -36050,7 +36386,8 @@ "MedicalTranscriptionJob.Specialty string `json:\"specialty,omitempty\"`", "MedicalTranscriptionJob.StartTime time.Time `json:\"startTime\"`", "MedicalTranscriptionJob.Tags map[string]string `json:\"tags,omitempty\"`", - "MedicalTranscriptionJob.TranscriptJSON []byte `json:\"-\"`", + "MedicalTranscriptionJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "MedicalTranscriptionJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "MedicalTranscriptionJob.TranscriptionJobStatus string `json:\"transcriptionJobStatus\"`", "MedicalTranscriptionJob.Type string `json:\"type,omitempty\"`", "MedicalTranscriptionSettings.ChannelIdentification bool `json:\"ChannelIdentification,omitempty\"`", @@ -36106,6 +36443,8 @@ "TranscriptionJob.LanguageCodes []LanguageCodeItem `json:\"languageCodes,omitempty\"`", "TranscriptionJob.LanguageIDSettings map[string]LanguageIDSettings `json:\"languageIdSettings,omitempty\"`", "TranscriptionJob.LanguageOptions []string `json:\"languageOptions,omitempty\"`", + "TranscriptionJob.ManagedOutputs map[string][]byte `json:\"managedOutputs,omitempty\"`", + "TranscriptionJob.ManagedToken string `json:\"managedToken,omitempty\"`", "TranscriptionJob.Media Media `json:\"media\"`", "TranscriptionJob.MediaFormat string `json:\"mediaFormat,omitempty\"`", "TranscriptionJob.MediaSampleRateHertz int32 `json:\"mediaSampleRateHertz,omitempty\"`", @@ -36113,13 +36452,14 @@ "TranscriptionJob.OutputBucketName string `json:\"outputBucketName,omitempty\"`", "TranscriptionJob.OutputEncryptionKMSKeyID string `json:\"outputEncryptionKMSKeyId,omitempty\"`", "TranscriptionJob.OutputKey string `json:\"outputKey,omitempty\"`", + "TranscriptionJob.RedactedTranscriptURI string `json:\"redactedTranscriptFileUri,omitempty\"`", "TranscriptionJob.Settings *TranscriptionSettings `json:\"settings,omitempty\"`", "TranscriptionJob.StartTime time.Time `json:\"startTime\"`", "TranscriptionJob.Subtitles *SubtitlesOutput `json:\"subtitles,omitempty\"`", "TranscriptionJob.Tags map[string]string `json:\"tags,omitempty\"`", "TranscriptionJob.ToxicityDetection []ToxicityDetectionSettings `json:\"toxicityDetection,omitempty\"`", - "TranscriptionJob.TranscriptJSON []byte `json:\"-\"`", - "TranscriptionJob.TranscriptText string `json:\"transcriptText\"`", + "TranscriptionJob.TranscriptFileURI string `json:\"transcriptFileUri,omitempty\"`", + "TranscriptionJob.TranscriptJSON []byte `json:\"transcriptJson,omitempty\"`", "TranscriptionSettings.ChannelIdentification bool `json:\"ChannelIdentification,omitempty\"`", "TranscriptionSettings.MaxAlternatives int32 `json:\"MaxAlternatives,omitempty\"`", "TranscriptionSettings.MaxSpeakerLabels int32 `json:\"MaxSpeakerLabels,omitempty\"`", diff --git a/podman-compose.yml b/podman-compose.yml index 9a8bd74722..9b0a99c4a3 100644 --- a/podman-compose.yml +++ b/podman-compose.yml @@ -9,6 +9,10 @@ services: container_name: gopherstack volumes: - gopherstack-data:/data + # Docker-backed engines (Lambda, ECS, ...): the image runs as UID 65532, so also mount the socket and + # join its group, e.g. `DOCKER_GID=$(stat -c %g /var/run/docker.sock) docker compose up`. + # - /var/run/docker.sock:/var/run/docker.sock + # group_add: ["${DOCKER_GID}"] volumes: gopherstack-data: diff --git a/services/apigateway/client_certificates.go b/services/apigateway/client_certificates.go index 18e9e2b1a4..c17fef6c89 100644 --- a/services/apigateway/client_certificates.go +++ b/services/apigateway/client_certificates.go @@ -21,7 +21,7 @@ func (b *InMemoryBackend) GenerateClientCertificate(input GenerateClientCertific Description: input.Description, PemEncodedCertificate: "-----BEGIN CERTIFICATE-----\nMIICpDCCAYwCCQDU...(mock)...\n-----END CERTIFICATE-----", CreatedDate: unixEpochTime{now}, - ExpirationDate: unixEpochTime{now.AddDate(0, 0, clientCertValidityDays)}, + ExpirationDate: unixEpochTime{now.UTC().AddDate(0, 0, clientCertValidityDays)}, } b.clientCertificates.Put(cert) diff --git a/services/appconfig/deployments.go b/services/appconfig/deployments.go index 436ccf5a9e..07d4c00340 100644 --- a/services/appconfig/deployments.go +++ b/services/appconfig/deployments.go @@ -394,12 +394,12 @@ func (b *InMemoryBackend) scheduleDeploymentReconcilerLocked() { defer ticker.Stop() for { - <-ticker.C + now := <-ticker.C - b.runDeploymentTicks() + b.runDeploymentTicks(now) b.mu.Lock("deploymentReconcile") - b.reconcileDeploymentsLocked() + b.reconcileDeploymentsLocked(now) done := len(b.deploymentTimers) == 0 b.mu.Unlock() @@ -413,9 +413,7 @@ func (b *InMemoryBackend) scheduleDeploymentReconcilerLocked() { // reconcileDeploymentsLocked advances every in-flight deployment whose // timer has elapsed by one progression step. Must be called under lock. -func (b *InMemoryBackend) reconcileDeploymentsLocked() { - now := time.Now() - +func (b *InMemoryBackend) reconcileDeploymentsLocked(now time.Time) { for key, timer := range b.deploymentTimers { if now.Before(timer.nextAt) { continue diff --git a/services/appconfig/extension_events.go b/services/appconfig/extension_events.go index f8e69e7ba6..55762969fe 100644 --- a/services/appconfig/extension_events.go +++ b/services/appconfig/extension_events.go @@ -349,9 +349,7 @@ type tickJob struct { } // collectTicksLocked gathers AT_DEPLOYMENT_TICK invocations for deployments whose timer is due. -func (b *InMemoryBackend) collectTicksLocked() []tickJob { - now := time.Now() - +func (b *InMemoryBackend) collectTicksLocked(now time.Time) []tickJob { var jobs []tickJob for key, timer := range b.deploymentTimers { @@ -384,10 +382,11 @@ func (b *InMemoryBackend) collectTicksLocked() []tickJob { } // runDeploymentTicks invokes AT_DEPLOYMENT_TICK actions synchronously and rolls a deployment back when an -// action errors or answers Directive ROLL_BACK. -func (b *InMemoryBackend) runDeploymentTicks() { +// action errors or answers Directive ROLL_BACK. now is shared with the reconcile pass so no deployment +// advances before its tick. +func (b *InMemoryBackend) runDeploymentTicks(now time.Time) { b.mu.RLock("collectTicks") - jobs := b.collectTicksLocked() + jobs := b.collectTicksLocked(now) b.mu.RUnlock() for _, job := range jobs { diff --git a/services/appconfig/whitebox_test.go b/services/appconfig/whitebox_test.go index c29886a6d9..2e812bd586 100644 --- a/services/appconfig/whitebox_test.go +++ b/services/appconfig/whitebox_test.go @@ -188,3 +188,36 @@ func TestDeploymentTimers_DrainToZero(t *testing.T) { assert.Equal(t, 0, deploymentTimerCount(), "every deployment timer must drain once its deployment completes") }) } + +func TestReconcileDeployments_AdvancesOnlyDueAtSharedNow(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend("000000000000", "us-east-1") + appID, envID, profileID, _ := seedDeployableConfig(t, b, []byte("v")) + + strategy, err := b.CreateDeploymentStrategy("slow", "", 5, 0, 50, "LINEAR", "NONE", nil) + require.NoError(t, err) + + _, err = b.StartDeployment(appID, envID, profileID, strategy.ID, "1", "", nil, nil, nil) + require.NoError(t, err) + + b.mu.Lock("test.reconcile") + defer b.mu.Unlock() + + require.Len(t, b.deploymentTimers, 1) + + var due time.Time + + for _, timer := range b.deploymentTimers { + due = timer.nextAt + } + + b.reconcileDeploymentsLocked(due.Add(-time.Nanosecond)) + + d, ok := b.deployments.Get(deploymentKey(appID, envID, 1)) + require.True(t, ok) + assert.Zero(t, d.PercentageComplete) + + b.reconcileDeploymentsLocked(due) + assert.InDelta(t, 50, d.PercentageComplete, 0) +} diff --git a/services/appsync/api_keys.go b/services/appsync/api_keys.go index 811b6e157a..226dda2e5a 100644 --- a/services/appsync/api_keys.go +++ b/services/appsync/api_keys.go @@ -56,7 +56,7 @@ func randomAPIKeyID() string { // documented 1-365-day bounds, returning ErrAPIKeyValidityOutOfBounds if it // falls outside them. func resolveAPIKeyExpiry(expires int64) (int64, error) { - now := time.Now() + now := time.Now().UTC() if expires <= 0 { return now.AddDate(0, 0, defaultAPIKeyExpiryDays).Unix(), nil diff --git a/services/azurearm/PARITY.md b/services/azurearm/PARITY.md index 4f9cfc1302..3e6e49e3c5 100644 --- a/services/azurearm/PARITY.md +++ b/services/azurearm/PARITY.md @@ -39,6 +39,7 @@ families: ContainerInstanceCRUD: {wire: ok, errors: ok, state: ok, persist: ok, note: "Microsoft.ContainerInstance containerGroups at api-version 2023-05-01 (rp_containerinstance*.go, aci_*.go), checked against armcontainerinstance/v2@v2.4.0 and driven by the SDK tests in aci_sdk_test.go: PUT (answers 200 once every container has started or failed, so the poller sees the final provisioningState), GET, PATCH (tags), DELETE, list by resource group and subscription, POST start/stop/restart (204), GET containers/{name}/logs?tail=N. Containers really run in the container runtime through pkgs/container (image pulled on demand; imageRegistryCredentials for the image's registry are used for an explicit authenticated pull, so images in the emulator's ACR work when the daemon can reach and trust it), with command (replaces the entrypoint), environment variables (secureValue is injected but never returned), resource limits (limits.cpu/memoryInGB cap the container; requests are validated but not enforced), ports, restartPolicy (Always restarts on any exit, OnFailure on a non-zero exit, Never never, with the platform's exponential back-off), volumes (emptyDir, secret, azureFile). instanceView is the live container state: currentState/previousState with exit code and times, restartCount, and Pulling/Pulled/Created/Started/BackOff/Failed events; the group state is Pending/Running/Succeeded/Failed/Stopped. ipAddress ports are published on 127.0.0.1 (ip is 127.0.0.1; the host port is reported in a PortPublished event) and dnsNameLabel yields

+
+ + +
+ + {#if error} + + {:else if loading} +

Loading…

+ {:else if apps.length === 0} +

No applications.

+ {:else} +
+ + + + + + {#each apps as a (a.id)} + + + + + + + + {/each} + +
NameApplication (client) IDCreatedSecretsActions
{a.appId}{when(a.created)}{a.secrets.length}
+
+ {/if} + + {#if detail} +
+

{detail.displayName}

+ {#if detailError}{/if} +
+
Object ID
{detail.id}
+
Redirect URIs
+
{detail.redirectUris.length ? detail.redirectUris.join(', ') : '—'}
+
API permissions
+
{detail.resourceAccess.length ? detail.resourceAccess.map((r) => `${r.resourceAppId} (${r.scopes} scopes, ${r.roles} roles)`).join('; ') : '—'}
+
App roles
+
{detail.appRoles.length ? detail.appRoles.map((r) => r.value || r.displayName).join(', ') : '—'}
+
Owners
+
{detail.owners.length ? detail.owners.map((o) => o.name || o.id).join(', ') : '—'}
+
+ +

Client secrets

+

A secret value is shown once when it is created. Afterwards only its hint and expiry are available.

+
addPassword(e, detail!)} data-testid="entra-secret-form"> + + + +
+ {#if detail.secrets.length === 0} +

No client secrets.

+ {:else} + + + + {#each detail.secrets as s (s.keyId)} + + + + + + + {/each} + +
DescriptionHintExpiresActions
{s.displayName || '—'}{s.hint ? `${s.hint}…` : '—'}{when(s.end)}{#if s.expired} (expired){/if}
+ {/if} +
+ {/if} + + + {#if shown} +
+

Copy this value now. It cannot be shown again.

+
+ {shown.value} + +
+
+
+ {/if} + +
diff --git a/ui/src/lib/components/azure/EntraDashboard.svelte b/ui/src/lib/components/azure/EntraDashboard.svelte new file mode 100644 index 0000000000..e72679c85b --- /dev/null +++ b/ui/src/lib/components/azure/EntraDashboard.svelte @@ -0,0 +1,71 @@ + + +
+ + + {#if loading} +
+ {:else} + {#if error} + + {/if} + (tab = id)} color="indigo" /> + + {#if SCOPED.has(tab) && subscription} +
+ + + {scope} +
+ {/if} + + + {/if} +
diff --git a/ui/src/lib/components/azure/EntraDirectory.svelte b/ui/src/lib/components/azure/EntraDirectory.svelte new file mode 100644 index 0000000000..ac0a72a4ad --- /dev/null +++ b/ui/src/lib/components/azure/EntraDirectory.svelte @@ -0,0 +1,153 @@ + + +
+ {#if error}{/if} + +
+

Users

+
+ + + + + +
+

The password is sent once and never shown or returned.

+ {#if loading} +

Loading…

+ {:else if users.length === 0} +

No users.

+ {:else} + + + + {#each users as u (u.id)} + + {/each} + +
User principal nameDisplay nameObject ID
{u.upn}{u.displayName}{u.id}
+ {/if} +
+ +
+

Groups

+
+ + +
+ {#if !loading && groups.length === 0} +

No groups.

+ {/if} + {#each groups as g (g.id)} +
+

{g.displayName} ({g.members.length} members)

+
    + {#each g.members as m (m.id)} +
  • + {m.name || m.id}{m.type}{m.detail ? ` · ${m.detail}` : ''} + +
    • + {/each} +
+
+ + +
+
+ {/each} +
+
diff --git a/ui/src/lib/components/azure/EntraLocks.svelte b/ui/src/lib/components/azure/EntraLocks.svelte new file mode 100644 index 0000000000..f7f0d76f5c --- /dev/null +++ b/ui/src/lib/components/azure/EntraLocks.svelte @@ -0,0 +1,81 @@ + + +
+
+ + + + +
+ + {#if error} + + {:else if loading} +

Loading…

+ {:else if locks.length === 0} +

No locks at this scope.

+ {:else} + + + + {#each locks as l (l.id || l.name)} + + + + + + + + {/each} + +
NameLock typeNotesApplies toActions
{l.name}{l.level}{l.notes || '—'}{l.id.split('/providers/Microsoft.Authorization/')[0]}
+ {/if} +
diff --git a/ui/src/lib/components/azure/EntraPolicy.svelte b/ui/src/lib/components/azure/EntraPolicy.svelte new file mode 100644 index 0000000000..8b468bbf72 --- /dev/null +++ b/ui/src/lib/components/azure/EntraPolicy.svelte @@ -0,0 +1,73 @@ + + +
+ {#if error} + + {:else if loading} +

Loading…

+ {:else} +
+

Assignments

+ {#if assigns.length === 0} +

No policy assignments at this scope.

+ {:else} + + + + {#each assigns as a (a.name)} + + {/each} + +
NameDefinitionScope
{a.displayName || a.name}{a.definition}{a.scope}
+ {/if} +
+
+
+

Definitions

+ + {shown.length} shown +
+ + + + {#each shown as d (d.name)} + + {/each} + +
NameTypeMode
{d.displayName || d.name}{d.policyType}{d.mode || '—'}
+
+ {/if} +
diff --git a/ui/src/lib/components/azure/EntraRoles.svelte b/ui/src/lib/components/azure/EntraRoles.svelte new file mode 100644 index 0000000000..fa212a5c54 --- /dev/null +++ b/ui/src/lib/components/azure/EntraRoles.svelte @@ -0,0 +1,117 @@ + + +
+
+ + + + +
+ + {#if error} + + {:else if loading} +

Loading…

+ {:else if assignments.length === 0} +

No role assignments at this scope.

+ {:else} +
+ + + + + + {#each assignments as a (a.name)} + {@const who = names.get(a.principalId)} + + + + + + + + {/each} + +
RolePrincipalTypeScopeActions
{roleNameOf(defs, a.roleDefinitionId)}{who?.name || a.principalId}{#if who}{a.principalId}{/if}{who?.type || a.principalType || '—'}{a.scope}
+
+ {/if} +
diff --git a/ui/src/lib/components/azure/EntraServicePrincipals.svelte b/ui/src/lib/components/azure/EntraServicePrincipals.svelte new file mode 100644 index 0000000000..0e0882f878 --- /dev/null +++ b/ui/src/lib/components/azure/EntraServicePrincipals.svelte @@ -0,0 +1,120 @@ + + +
+
+ + + +
+ + {#if error} + + {:else if loading} +

Loading…

+ {:else if sps.length === 0} +

No service principals.

+ {:else} +
+ + + + + + {#each sps as s (s.id)} + + + + + + + {/each} + +
NameApplication IDTypeEnabled
{s.appId}{s.type || '—'}{s.enabled ? 'Yes' : 'No'}
+
+ {/if} + + {#if selected} +
+

App role assignments

+ {#if assignError}{/if} + {#if assignments.length === 0} +

No app role assignments.

+ {:else} + + + + {#each assignments as a (a.id)} + + {/each} + +
PrincipalTypeResourceRole ID
{a.principal}{a.principalType}{a.resource}{a.appRoleId}
+ {/if} +
+ {/if} +
diff --git a/ui/src/lib/components/azure/EntraTabBody.svelte b/ui/src/lib/components/azure/EntraTabBody.svelte new file mode 100644 index 0000000000..fc74e834db --- /dev/null +++ b/ui/src/lib/components/azure/EntraTabBody.svelte @@ -0,0 +1,25 @@ + + +{#if tab === 'apps'} + +{:else if tab === 'sps'} + +{:else if tab === 'directory'} + +{:else if tab === 'roles' && scope} + {#key scope}{/key} +{:else if tab === 'locks' && scope} + {#key scope}{/key} +{:else if tab === 'policy' && scope} + {#key scope}{/key} +{/if} diff --git a/ui/src/lib/components/azure/NetworkAppGateways.svelte b/ui/src/lib/components/azure/NetworkAppGateways.svelte new file mode 100644 index 0000000000..76e3cd63f2 --- /dev/null +++ b/ui/src/lib/components/azure/NetworkAppGateways.svelte @@ -0,0 +1,114 @@ + + +
+ {#if error}{/if} + {#if loading && gateways.length === 0} +
+ {:else if gateways.length === 0} +

No application gateways in this resource group.

+ {/if} + {#each gateways as g (g.id)} +
+
+

{g.name} + {g.operationalState || '—'} +

+
+ + + +
+
+

{g.sku} / {g.tier} · capacity {g.capacity || '—'} · {g.location}

+
+

Listeners

+ {#each g.listeners as l (l.name)}

{l.name}: {l.protocol} :{l.port}{l.host ? ` (${l.host})` : ''}

{:else}

None

{/each} +
+

Backend pools

+ {#each g.pools as p (p.name)}

{p.name}: {p.addresses.join(', ') || 'no addresses'}

{:else}

None

{/each} +
+

Backend settings

+ {#each g.settings as s (s.name)}

{s.name}: {s.protocol} :{s.port}

{:else}

None

{/each} +
+

Routing rules

+ {#each g.rules as r (r.name)}

{r.priority ? `${r.priority} ` : ''}{r.name}: {r.type} {r.listener} → {r.pool || '—'}

{:else}

None

{/each} +
+

Health probes

+ {#each g.probes as p (p.name)}

{p.name}: {p.protocol} {p.host}{p.path}

{:else}

None

{/each} +
+
+ {#if healthError[g.name]}{/if} + {#if health[g.name]} +
+

Backend health

+ {#each health[g.name] as pool (pool.pool + pool.settings)} + {#each pool.servers as s (s.address)} +

{pool.pool} / {pool.settings} · {s.address}: {s.health} {s.log}

+ {/each} + {:else} +

No backend servers.

+ {/each} +
+ {/if} +
+ {/each} +
diff --git a/ui/src/lib/components/azure/NetworkFirewalls.svelte b/ui/src/lib/components/azure/NetworkFirewalls.svelte new file mode 100644 index 0000000000..2f0b8fb165 --- /dev/null +++ b/ui/src/lib/components/azure/NetworkFirewalls.svelte @@ -0,0 +1,127 @@ + + +
+ {#if error}{/if} + {#if loading && firewalls.length + policies.length + ipGroups.length === 0} +
+ {:else} +
+

Azure Firewalls

+ {#each firewalls as f (f.id)} +
+

{f.name} {f.sku} / {f.tier} · {f.location}

+

Threat intelligence: {f.threatIntel || '—'}

+

Firewall policy: {f.policy || 'none (classic rules)'}

+ {#each f.ipConfigs as c (c.name)}

IP configuration {c.name}: subnet {c.subnet || '—'}, public IP {c.publicIp || '—'}, private {c.privateIp || '—'}

{/each} +
+ {:else} +

No firewalls in this resource group.

+ {/each} +
+ +
+

Firewall policies

+ {#if policies.length === 0} +

No firewall policies in this resource group.

+ {:else} +
+ + + + {#each policies as p (p.id)} + + + + + + + {/each} + +
PolicyTierThreat intelBase policy
{p.tier || '—'}{p.threatIntel || '—'}{p.basePolicy || '—'}
+
+ {#if detailError}{/if} + {#each groups as g (g.name)} +
+

{g.name} priority {g.priority}

+ {#each g.collections as c (c.name)} +
+

{c.priority} · {c.name} {c.type} · {c.action || '—'}

+ {#each c.rules as r (r.name)}

{r.name} [{r.type}] {r.summary}

{/each} +
+ {:else} +

No rule collections.

+ {/each} +
+ {:else} + {#if !detailError && selected}

This policy has no rule collection groups.

{/if} + {/each} + {/if} +
+ +
+

IP groups

+ {#each ipGroups as g (g.id)} +

{g.name}: {g.addresses.join(', ') || 'no addresses'}{g.firewalls.length ? ` · used by ${g.firewalls.join(', ')}` : ''}

+ {:else} +

No IP groups in this resource group.

+ {/each} +
+ {/if} +
diff --git a/ui/src/lib/components/azure/NetworkLoadBalancers.svelte b/ui/src/lib/components/azure/NetworkLoadBalancers.svelte new file mode 100644 index 0000000000..83adf1338e --- /dev/null +++ b/ui/src/lib/components/azure/NetworkLoadBalancers.svelte @@ -0,0 +1,68 @@ + + +
+ {#if error}{/if} + {#if loading && lbs.length === 0} +
+ {:else if lbs.length === 0} +

No load balancers in this resource group.

+ {/if} + {#each lbs as lb (lb.id)} +
+
+

{lb.name} {lb.sku} · {lb.location}

+ +
+
+

Frontends

+ {#each lb.frontends as f (f.name)}

{f.name}: {f.publicIp ? `public ${f.publicIp}` : f.privateIp || '—'}

{:else}

None

{/each} +
+

Backend pools

+ {#each lb.pools as p (p.name)}

{p.name}: {p.members} member(s)

{:else}

None

{/each} +
+

Health probes

+ {#each lb.probes as p (p.name)}

{p.name}: {p.protocol} :{p.port}{p.path}

{:else}

None

{/each} +
+

Load balancing rules

+ {#each lb.rules as r (r.name)}

{r.name}: {r.protocol} {dash(r.frontendPort)} → {dash(r.backendPort)}

{:else}

None

{/each} +
+
+
+ {/each} +
diff --git a/ui/src/lib/components/azure/NetworkNsgs.svelte b/ui/src/lib/components/azure/NetworkNsgs.svelte new file mode 100644 index 0000000000..d8e9912ddf --- /dev/null +++ b/ui/src/lib/components/azure/NetworkNsgs.svelte @@ -0,0 +1,145 @@ + + +
+ {#if error} + + {/if} + {#if loading && nsgs.length === 0} +
+ {:else if nsgs.length === 0} +

No network security groups in this resource group.

+ {:else if current} +
+ + +
+ +
+ + + + {#each current.rules as r (r.name)} + + + + + + + + + + + + + {:else} + + {/each} + {#each current.defaultRules as r (r.name)} + + + + + + + + + + + + + {/each} + +
PriorityNameDirectionAccessProtocolSourceSource portsDestinationDest. portsActions
{r.priority}{r.name}{r.direction}{r.access}{r.protocol}{join(r.sources)}{join(r.sourcePorts)}{join(r.destinations)}{join(r.destinationPorts)}
No custom rules.
{r.priority}{r.name}{r.direction}{r.access}{r.protocol}{join(r.sources)}{join(r.sourcePorts)}{join(r.destinations)}{join(r.destinationPorts)}default
+
+ +
+

Add security rule to {current.name}

+
+ + + + + + + + + + + +
+ {#if draft.name && draftError}

{draftError}

{/if} +
+ {/if} +
diff --git a/ui/src/lib/components/azure/NetworkPublicIps.svelte b/ui/src/lib/components/azure/NetworkPublicIps.svelte new file mode 100644 index 0000000000..730429805e --- /dev/null +++ b/ui/src/lib/components/azure/NetworkPublicIps.svelte @@ -0,0 +1,57 @@ + + +
+ {#if error}{/if} + {#if loading && ips.length === 0} +
+ {:else if ips.length === 0} +

No public IP addresses in this resource group.

+ {:else} +
+ + + + {#each ips as ip (ip.id)} + + + + + + + + + {/each} + +
NameLocationSKUAllocationVersionIP address
{ip.name}{ip.location}{ip.sku || '—'}{ip.tier ? ` (${ip.tier})` : ''}{ip.allocation || '—'}{ip.version || '—'}{ip.address || 'Not assigned'}
+
+ {/if} +
diff --git a/ui/src/lib/components/azure/NetworkTabBody.svelte b/ui/src/lib/components/azure/NetworkTabBody.svelte new file mode 100644 index 0000000000..bea0d4858c --- /dev/null +++ b/ui/src/lib/components/azure/NetworkTabBody.svelte @@ -0,0 +1,24 @@ + + +{#if tab === 'vnets'} + +{:else if tab === 'nsgs'} + +{:else if tab === 'ips'} + +{:else if tab === 'lbs'} + +{:else if tab === 'agws'} + +{:else if tab === 'firewalls'} + +{/if} diff --git a/ui/src/lib/components/azure/NetworkVnets.svelte b/ui/src/lib/components/azure/NetworkVnets.svelte new file mode 100644 index 0000000000..c92a8d367b --- /dev/null +++ b/ui/src/lib/components/azure/NetworkVnets.svelte @@ -0,0 +1,190 @@ + + +
+ {#if error} + + {/if} + {#if loading && vnets.length === 0} +
+ {:else if vnets.length === 0} +

No virtual networks in this resource group.

+ {:else} +
+ + + + {#each vnets as v (v.id)} + + + + + + + + {/each} + +
NameLocationAddress spaceSubnetsActions
{v.location}{v.addressSpace.join(', ') || '—'}{v.subnets.length}
+
+ {/if} + + {#if current} +
+

Subnets of {current.name}

+ {#if current.subnets.length === 0} +

This virtual network has no subnets.

+ {:else} +
+ + + + {#each current.subnets as s (s.name)} + + + + + + + + + {/each} + +
SubnetPrefixDelegationsNSGNAT gatewayActions
{s.name}{s.addressPrefix}{s.delegations.map((d) => d.service).join(', ') || '—'}{s.nsg || '—'}{s.natGateway || '—'}
+
+ {/if} + +
createSubnet(e, current)} data-testid="net-subnet-form"> + + + + + + +
+ {#if subnetDraft.name || subnetDraft.prefix} + {#if subnetError}

{subnetError}

{/if} + {/if} +
+ {/if} + +
+

Create virtual network

+
+ + + + + + + +
+ {#if vnetDraft.name && vnetError}

{vnetError}

{/if} +
+
diff --git a/ui/src/lib/components/azure/NetworkingDashboard.svelte b/ui/src/lib/components/azure/NetworkingDashboard.svelte new file mode 100644 index 0000000000..c74b10cea1 --- /dev/null +++ b/ui/src/lib/components/azure/NetworkingDashboard.svelte @@ -0,0 +1,71 @@ + + +
+ + + {#if loading} +
+ {:else if error} + + {:else if groups.length === 0} +

No resource groups. Create one in Resource Manager first.

+ {:else} +
+ Subscription{subscription} + + +
+ (tab = id)} color="sky" /> + {#key `${group}:${reloads}`} + + {/key} + {/if} +
diff --git a/ui/src/lib/components/gcp/ComputeArmor.svelte b/ui/src/lib/components/gcp/ComputeArmor.svelte new file mode 100644 index 0000000000..f03f569538 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeArmor.svelte @@ -0,0 +1,50 @@ + + + load()}> + {#snippet detail(row)} + {@const p = byName.get(row.id)} + {#if p} + + + + {#each p.rules as r (r.priority)} + + {/each} + +
PriorityActionMatchDescriptionPreview
{r.priority}{r.action}{r.match || '—'}{r.description || '—'}{r.preview ? 'yes' : 'no'}
+ {/if} + {/snippet} + diff --git a/ui/src/lib/components/gcp/ComputeDashboard.svelte b/ui/src/lib/components/gcp/ComputeDashboard.svelte index e3b977e184..68efc73cb5 100644 --- a/ui/src/lib/components/gcp/ComputeDashboard.svelte +++ b/ui/src/lib/components/gcp/ComputeDashboard.svelte @@ -3,259 +3,85 @@ import { Server } from 'lucide-svelte'; import PageHeader from '#lib/components/PageHeader.svelte'; import Tabs from '#lib/components/Tabs.svelte'; + import ComputeTabContent from '#lib/components/gcp/ComputeTabContent.svelte'; import GcpScopeBar from '#lib/components/gcp/GcpScopeBar.svelte'; - import GcpResourceTable, { type ResourceRow } from '#lib/components/gcp/GcpResourceTable.svelte'; - import { confirmDestructive } from '#lib/confirm-dialog.ts'; - import { attempt, GCP_INPUT, GCP_MINI } from '#lib/gcp-action.ts'; - import { - ALL_ZONES, - deleteAddress, - deleteDisk, - deleteInstance, - deleteSnapshot, - instanceAction, - labelText, - listAddresses, - listDisks, - listFirewalls, - listInstances, - listNetworks, - listOperations, - listRouters, - listRoutes, - listSnapshots, - listSubnetworks, - listZoneNames, - readStoredComputeScope, - regionOfZone, - writeStoredComputeScope, - type Address, - type ComputeOperation, - type Disk, - type Firewall, - type Instance, - type InstanceAction, - type Network, - type Router, - type Route, - type Snapshot, - type Subnetwork - } from '#lib/gcpcompute.ts'; - import { describeGcpError } from '#lib/gcprest.ts'; + import { GCP_INPUT } from '#lib/gcp-action.ts'; + import { ALL_REGIONS, ALL_ZONES, listRegionNames, listZoneNames, readStoredComputeScope, regionOfZone, writeStoredComputeScope } from '#lib/gcpcompute.ts'; const TABS = [ { id: 'instances', label: 'Instances' }, - { id: 'disks', label: 'Disks' }, - { id: 'snapshots', label: 'Snapshots' }, + { id: 'storage', label: 'Disks, images & snapshots' }, { id: 'networks', label: 'Networks' }, - { id: 'firewalls', label: 'Firewalls' }, - { id: 'routes', label: 'Routes' }, - { id: 'routers', label: 'Routers / NAT' }, - { id: 'addresses', label: 'Addresses' }, + { id: 'lb', label: 'Load balancing' }, + { id: 'armor', label: 'Cloud Armor' }, { id: 'operations', label: 'Operations' } ]; const stored = readStoredComputeScope(); let project = $state(stored.project); let zone = $state(stored.zone); + let region = $state(stored.region); let zones = $state([]); + let regions = $state([]); let tab = $state('instances'); - let loading = $state(true); - let error = $state(''); + let refresh = $state(0); - let instances = $state([]); - let disks = $state([]); - let snapshots = $state([]); - let networks = $state([]); - let subnets = $state([]); - let firewalls = $state([]); - let routes = $state([]); - let routers = $state([]); - let addresses = $state([]); - let operations = $state([]); + const zoneChoices = $derived(region ? zones.filter((z) => regionOfZone(z) === region) : zones); - const region = $derived(zone ? regionOfZone(zone) : ''); - const inZone = (z: string) => !zone || z === zone; - const inRegion = (r: string) => !region || r === region || r === 'global'; - - async function load(quiet = false) { - if (!quiet) loading = true; - try { - switch (tab) { - case 'instances': - instances = await listInstances(project); - break; - case 'disks': - disks = await listDisks(project); - break; - case 'snapshots': - snapshots = await listSnapshots(project); - break; - case 'networks': - [networks, subnets] = await Promise.all([listNetworks(project), listSubnetworks(project)]); - break; - case 'firewalls': - firewalls = await listFirewalls(project); - break; - case 'routes': - routes = await listRoutes(project); - break; - case 'routers': - routers = await listRouters(project); - break; - case 'addresses': - addresses = await listAddresses(project); - break; - default: - operations = await listOperations(project); - } - error = ''; - } catch (e) { - error = describeGcpError(e); - } finally { - loading = false; - } - } - - async function loadZones() { + async function loadScope() { try { - zones = await listZoneNames(project); + [zones, regions] = await Promise.all([listZoneNames(project), listRegionNames(project)]); } catch { zones = []; + regions = []; } } + function persist() { + writeStoredComputeScope(project, zone, region); + } + function applyScope(p: string) { project = p; - writeStoredComputeScope(project, zone); - void loadZones(); - void load(); + persist(); + void loadScope(); } function pickZone(event: Event) { zone = (event.currentTarget as HTMLSelectElement).value; - writeStoredComputeScope(project, zone); + if (zone) region = regionOfZone(zone); + persist(); } - function selectTab(id: string) { - tab = id; - void load(); + function pickRegion(event: Event) { + region = (event.currentTarget as HTMLSelectElement).value; + if (zone && regionOfZone(zone) !== region) zone = ALL_ZONES; + persist(); } - async function run(action: InstanceAction, i: Instance) { - if (await attempt(() => instanceAction(project, i.zone, i.name, action), `${action[0].toUpperCase()}${action.slice(1)} requested for ${i.name}`, `Failed to ${action} ${i.name}`)) { - await load(true); - for (const ms of [1500, 5000, 12000]) setTimeout(() => void load(true), ms); - } - } - - async function removeWith(kind: string, name: string, fn: () => Promise) { - if (!(await confirmDestructive({ title: `Delete ${kind}`, message: `Delete ${kind} "${name}"?` }))) return; - if (await attempt(fn, `Deleted ${name}`, `Failed to delete ${name}`)) await load(true); - } - - const instById = $derived(new Map(instances.map((i) => [`${i.zone}/${i.name}`, i]))); - const diskById = $derived(new Map(disks.map((d) => [`${d.zone}/${d.name}`, d]))); - const addrById = $derived(new Map(addresses.map((a) => [`${a.scope}/${a.name}`, a]))); - - const instanceRows = $derived( - instances - .filter((i) => inZone(i.zone)) - .map((i) => ({ - id: `${i.zone}/${i.name}`, - cells: [i.name, i.zone, i.status, i.machineType, i.internalIps.join(', '), i.externalIps.join(', '), i.disks.map((d) => d.name).join(', '), labelText(i.labels), i.metadataKeys.join(', ')], - raw: i.raw - })) - ); - const diskRows = $derived( - disks - .filter((d) => inZone(d.zone)) - .map((d) => ({ id: `${d.zone}/${d.name}`, cells: [d.name, d.zone, d.status, `${d.sizeGb} GB`, d.type, d.users.join(', '), labelText(d.labels)], raw: d.raw })) - ); - const snapshotRows = $derived(snapshots.map((s) => ({ id: s.name, cells: [s.name, s.status, s.sourceDisk, `${s.sizeGb} GB`, s.created], raw: s.raw }))); - const networkRows = $derived(networks.map((n) => ({ id: n.name, cells: [n.name, n.mode, n.routing, String(subnets.filter((s) => s.network === n.name).length), n.mtu ? String(n.mtu) : '', n.created], raw: n.raw }))); - const firewallRows = $derived( - firewalls.map((f) => ({ id: f.name, cells: [f.name, f.network, f.direction, String(f.priority), f.rules.join(', '), f.ranges.join(', '), f.targets.join(', '), f.disabled ? 'disabled' : 'enabled'], raw: f.raw })) - ); - const routeRows = $derived(routes.map((r) => ({ id: r.name, cells: [r.name, r.network, r.destRange, String(r.priority), r.nextHop, r.tags.join(', ')], raw: r.raw }))); - const routerRows = $derived( - routers - .filter((r) => inRegion(r.region)) - .map((r) => ({ id: `${r.region}/${r.name}`, cells: [r.name, r.region, r.network, r.nats.join('; '), r.asn ? String(r.asn) : ''], raw: r.raw })) - ); - const addressRows = $derived( - addresses - .filter((a) => inRegion(a.scope)) - .map((a) => ({ id: `${a.scope}/${a.name}`, cells: [a.name, a.scope, a.address, a.type, a.status, a.purpose, a.users.join(', ')], raw: a.raw })) - ); - const operationRows = $derived( - operations - .filter((o) => !zone || o.scope === zone || o.scope === region || o.scope === 'global') - .map((o) => ({ id: o.name, cells: [o.type, o.target, o.scope, o.progress === 100 ? o.status : `${o.status} ${o.progress}%`, o.started, o.ended, o.error], raw: o.raw })) - ); - - onMount(() => { - void loadZones(); - void load(); - }); + onMount(() => void loadScope());
- load()} /> + (refresh += 1)} />
+
- + (tab = id)} color="blue" /> - {#if tab === 'instances'} - load()} onDelete={(r) => { const i = instById.get(r.id); if (i) void removeWith('instance', i.name, () => deleteInstance(project, i.zone, i.name)); }}> - {#snippet actions(row: ResourceRow)} - {@const i = instById.get(row.id)} - {#if i} - - - - {/if} - {/snippet} - - {:else if tab === 'disks'} - load()} onDelete={(r) => { const d = diskById.get(r.id); if (d) void removeWith('disk', d.name, () => deleteDisk(project, d.zone, d.name)); }} /> - {:else if tab === 'snapshots'} - load()} onDelete={(r) => void removeWith('snapshot', r.id, () => deleteSnapshot(project, r.id))} /> - {:else if tab === 'networks'} - load()}> - {#snippet detail(row: ResourceRow)} - {@const subs = subnets.filter((s) => s.network === row.id)} - {#if subs.length === 0} -

No subnetworks

- {:else} - - - - {#each subs as s (`${s.region}/${s.name}`)} - - {/each} - -
SubnetworkRegionRangeGatewayPrivate Google access
{s.name}{s.region}{s.cidr}{s.gateway || '—'}{s.privateGoogleAccess ? 'on' : 'off'}
- {/if} - {/snippet} - - {:else if tab === 'firewalls'} - load()} /> - {:else if tab === 'routes'} - load()} /> - {:else if tab === 'routers'} - load()} /> - {:else if tab === 'addresses'} - load()} onDelete={(r) => { const a = addrById.get(r.id); if (a) void removeWith('address', a.name, () => deleteAddress(project, a.scope, a.name)); }} /> - {:else} - load()} /> - {/if} +
diff --git a/ui/src/lib/components/gcp/ComputeInstanceDetail.svelte b/ui/src/lib/components/gcp/ComputeInstanceDetail.svelte new file mode 100644 index 0000000000..25f2365651 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeInstanceDetail.svelte @@ -0,0 +1,75 @@ + + +
+
+
+

Disks

+ {#if instance.disks.length === 0}

No disks

{:else} + + + + {#each instance.disks as d (d.name)} + + {/each} + +
NameBootModeSize
{d.name}{d.boot ? 'boot' : '—'}{d.mode || '—'}{d.sizeGb ? `${d.sizeGb} GB` : '—'}
+ {/if} +
+
+

Network interfaces

+ {#if instance.nics.length === 0}

No network interfaces

{:else} + + + + {#each instance.nics as n, i (i)} + + {/each} + +
InterfaceNetworkSubnetworkInternal IPExternal IP
{n.name || `nic${i}`}{n.network || '—'}{n.subnetwork || '—'}{n.internalIp || '—'}{n.externalIps.join(', ') || '—'}
+ {/if} +
+
+ {#if labels}

Labels {labels}

{/if} + {#if instance.tags.length > 0}

Network tags {instance.tags.join(', ')}

{/if} +
+

Metadata (values are hidden; a key can be revealed on request, SSH keys never)

+ {#if instance.metadata.length === 0}

No metadata

{:else} +
    + {#each instance.metadata as m (m.key)} +
  • + {m.key} + {m.length} bytes + {#if m.key in revealed} + + 
    {revealed[m.key]}
    + {:else if m.revealable} + + {:else} + never shown + {/if} +
    • + {/each} +
+ {/if} +
+
diff --git a/ui/src/lib/components/gcp/ComputeInstanceForm.svelte b/ui/src/lib/components/gcp/ComputeInstanceForm.svelte new file mode 100644 index 0000000000..b26d5cb286 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeInstanceForm.svelte @@ -0,0 +1,120 @@ + + +
{ e.preventDefault(); void submit(); }} data-testid="compute-instance-form"> +
+ + + + + + + + +
+ {#if loadError}{/if} +
+ + +
+
diff --git a/ui/src/lib/components/gcp/ComputeInstances.svelte b/ui/src/lib/components/gcp/ComputeInstances.svelte new file mode 100644 index 0000000000..e82bae6c5f --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeInstances.svelte @@ -0,0 +1,104 @@ + + +
+
+ +
+ {#if creating} + (creating = false)} /> + {/if} + load()} onDelete={(r) => { const i = byId.get(r.id); if (i) void remove(i); }}> + {#snippet actions(row: ResourceRow)} + {@const i = byId.get(row.id)} + {#if i} + + + + + + {/if} + {/snippet} + {#snippet detail(row: ResourceRow)} + {@const i = byId.get(row.id)} + {#if i}{/if} + {/snippet} + +
diff --git a/ui/src/lib/components/gcp/ComputeLoadBalancing.svelte b/ui/src/lib/components/gcp/ComputeLoadBalancing.svelte new file mode 100644 index 0000000000..aae1befe4b --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeLoadBalancing.svelte @@ -0,0 +1,76 @@ + + +
+ + {#if tab === 'backends'} + load()} /> + {:else if tab === 'urlmaps'} + load()} /> + {:else if tab === 'proxies'} + load()} /> + {:else} + load()} /> + {/if} +
diff --git a/ui/src/lib/components/gcp/ComputeNetworks.svelte b/ui/src/lib/components/gcp/ComputeNetworks.svelte new file mode 100644 index 0000000000..0f1858f7e4 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeNetworks.svelte @@ -0,0 +1,134 @@ + + +
+ + {#if tab === 'vpcs'} + load()}> + {#snippet detail(row: ResourceRow)} + {@const subs = subnets.filter((s) => s.network === row.id)} + {#if subs.length === 0} +

No subnetworks

+ {:else} + + + + {#each subs as s (`${s.region}/${s.name}`)} + + {/each} + +
SubnetworkRegionRangeGatewayPrivate Google access
{s.name}{s.region}{s.cidr}{s.gateway || '—'}{s.privateGoogleAccess ? 'on' : 'off'}
+ {/if} + {/snippet} + + {:else if tab === 'subnets'} + load()} /> + {:else if tab === 'firewalls'} + load()} /> + {:else if tab === 'routes'} + load()} /> + {:else if tab === 'peering'} + load()} /> + {:else if tab === 'routers'} + load()} /> + {:else} + load()} onDelete={(r) => { const a = addrById.get(r.id); if (a) void removeAddress(a); }} /> + {/if} +
diff --git a/ui/src/lib/components/gcp/ComputeOperations.svelte b/ui/src/lib/components/gcp/ComputeOperations.svelte new file mode 100644 index 0000000000..577555d1c1 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeOperations.svelte @@ -0,0 +1,55 @@ + + + load()} /> diff --git a/ui/src/lib/components/gcp/ComputeStorage.svelte b/ui/src/lib/components/gcp/ComputeStorage.svelte new file mode 100644 index 0000000000..8c7b74ba19 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeStorage.svelte @@ -0,0 +1,102 @@ + + +
+ + {#if tab === 'disks'} + {#if snapOf} +
{ e.preventDefault(); void snapshot(); }} data-testid="compute-snapshot-form"> + + + +
+ {/if} + load()} onDelete={(r) => { const d = diskById.get(r.id); if (d) void removeDisk(d); }}> + {#snippet actions(row: ResourceRow)} + {@const d = diskById.get(row.id)} + {#if d}{/if} + {/snippet} + + {:else if tab === 'images'} + load()} /> + {:else} + load()} onDelete={(r) => void removeSnapshot(r.id)} /> + {/if} +
diff --git a/ui/src/lib/components/gcp/ComputeTabContent.svelte b/ui/src/lib/components/gcp/ComputeTabContent.svelte new file mode 100644 index 0000000000..9253d3ab15 --- /dev/null +++ b/ui/src/lib/components/gcp/ComputeTabContent.svelte @@ -0,0 +1,25 @@ + + +{#if tab === 'instances'} + +{:else if tab === 'storage'} + +{:else if tab === 'networks'} + +{:else if tab === 'lb'} + +{:else if tab === 'armor'} + +{:else} + +{/if} diff --git a/ui/src/lib/components/oci/DatabaseDashboard.svelte b/ui/src/lib/components/oci/DatabaseDashboard.svelte new file mode 100644 index 0000000000..c288875364 --- /dev/null +++ b/ui/src/lib/components/oci/DatabaseDashboard.svelte @@ -0,0 +1,410 @@ + + +
+ + + {#if error && compartments.length === 0} + + {:else if loading} +
+ {:else} +
+ + Engine: {engine || 'unknown'} + +
+ + {#if error}{/if} + + {#if showCreate} +
{ e.preventDefault(); void create(); }} data-testid="db-create-form"> + + + + + + + + +
+ + {#if invalid && (fDbName || fPassword)}{invalid}{/if} +
+
+ {/if} + +
+ + + + {#each databases as d (d.id)} + + + + + + + + + + + {:else} + + {/each} + +
Display nameDB nameWorkloadStateComputeStorageFree tiermTLS required
{d.dbName}{d.workload}{d.state}{compute(d)}{d.storageTBs} TB{d.freeTier ? 'yes' : 'no'}{d.mtlsRequired ? 'yes' : 'no'}
No Autonomous Databases in this compartment. Create one above.
+
+ + {#if db && detail} +
+
+

{detail.name} {detail.state}

+
+ {#each ['start', 'stop', 'restart'] as const as a (a)} + + {/each} + +
+
+ {#if detail.details}

{detail.details}

{/if} + {#if detailError}{/if} +
+
OCID
{detail.id}
+
Version
{detail.version || '—'}
+
Backup retention
{detail.backupRetentionDays} days
+
Private endpoint
{detail.privateEndpoint || 'none (public)'}
+
Created
{when(detail.created)}
+
+
{ e.preventDefault(); void rename(db); }}> + + +
+ + (tab = id)} color="orange" /> + + {#if tab === 'connections'} +
+ {#if detail.profiles.length === 0} +

No connection strings yet. They appear once the database is available.

+ {:else} +
+ + + + {#each detail.profiles as p, i (i)} + + + + + + + + + {/each} + +
ProfileGroupProtocolTLSConnection string
{p.name}{p.consumerGroup}{p.protocol}{p.tlsAuthentication}{p.value}
+
+ {/if} +
+

Connection URLs

+
    + {#each Object.entries(detail.connectionUrls) as [k, v] (k)}
  • {k} {v}
    • {:else}
  • None
    • {/each} +
+
+
+ {:else if tab === 'access'} +
+
{ e.preventDefault(); void saveAccess(db); }} data-testid="db-acl-form"> +

Access control list

+ + + {#if aclInvalid !== undefined}

Invalid entry: {aclInvalid}

{/if} + +
+
{ e.preventDefault(); void saveCompute(db); }} data-testid="db-compute-form"> +

Compute

+ + {#if detail.freeTier}

Always Free databases have a fixed shape.

{/if} + +
+
+ {:else if tab === 'backups'} +
+
{ e.preventDefault(); void backupNow(db); }}> + + +
+
+ + + + {#each backups as b (b.id)} + + + + + + + + {:else} + + {/each} + +
NameTypeStateStartedAvailable until
{b.name}{b.automatic ? ' (automatic)' : ''}{b.type}{b.state}{when(b.started)}{when(b.availableTill)}
No backups
+
+
+ {:else} +
{ e.preventDefault(); void wallet(db); }} data-testid="db-wallet-form"> +

The wallet zip holds the client certificate for mutual TLS, encrypted with the password you choose here. The password is not stored or shown again.

+ + +
+ {/if} +
+ {/if} + {/if} +
diff --git a/ui/src/lib/components/oci/DnsDashboard.svelte b/ui/src/lib/components/oci/DnsDashboard.svelte new file mode 100644 index 0000000000..2c2b7b2972 --- /dev/null +++ b/ui/src/lib/components/oci/DnsDashboard.svelte @@ -0,0 +1,514 @@ + + +
+ + + {#if error && compartments.length === 0} + + {:else if loading} +
+ {:else} +
+ + + {dataPlane.enabled ? `Data plane: port ${dataPlane.port}` : 'Data plane: off'} + +
+ + {#if error}{/if} + + (tab = id)} color="orange" /> + + {#if tab === 'zones'} +
+ +
+ + {#if showCreate} +
{ e.preventDefault(); void create(); }} data-testid="dns-create-form"> + + + {#if zScope === 'PRIVATE'} + + {/if} +
+ + {#if zoneInvalid && zName}{zoneInvalid}{/if} +
+
+ {/if} + +
+ + + + {#each zones as z (z.id)} + + + + + + + + + {:else} + + {/each} + +
NameTypeScopeSerialCreatedActions
{z.type}{z.scope}{z.viewId ? ` · ${viewName(z.viewId)}` : ''}{z.serial}{when(z.created)}
No zones in this compartment. Create one above.
+
+ + {#if zone} +
+

{zone.name}

+
+
OCID
{zone.id}
+
Name servers
{zone.nameservers.join(', ') || '—'}
+
+ {#if detailError}{/if} + +
+ + + {#each rrsets as set (set.domain + '|' + set.rtype)} + + {#each set.records as r, i (r.rdata)} + + + + + + + + {/each} + + {:else} + + {/each} +
DomainTypeTTLDataActions
{i === 0 ? r.domain : ''}{i === 0 ? r.rtype : ''}{i === 0 ? r.ttl : ''}{r.rdata} + + +
No records
+
+ +
{ e.preventDefault(); void saveRecord(); }} data-testid="dns-record-form"> +

{editing ? `Edit ${editing.rtype} record` : 'Add record'}

+ + + + +
+ + {#if editing}{/if} + {#if recordInvalid && (rData || editing)}{recordInvalid}{/if} +
+
+
+ {/if} + +
+

Try it

+ {#if !dataPlane.enabled} +

The DNS data plane is off (OCI_DNS_PORT_DNS is 0), so queries are unavailable.

+ {:else} +

Runs a query against the emulator's DNS server on 127.0.0.1:{dataPlane.port}. Only GLOBAL zones answer.

+
{ e.preventDefault(); void runQuery(); }}> + + + +
+ {#if queryError}{/if} + {#if answer} +
+

Status {answer.rcode}{answer.authoritative ? ' · authoritative' : ''} · {answer.name} {answer.type} from {answer.server}

+
+ + + + {#each answer.answer as rr, i (i)} + + + + + + + {:else} + + {/each} + +
NameTypeTTLData
{rr.name}{rr.type}{rr.ttl}{rr.data}
Empty answer section
+
+
+ {/if} + {/if} +
+ {:else if tab === 'views'} +
+ + + + {#each views as v (v.id)} + + {:else} + + {/each} + +
NameStateProtectedCreatedOCID
{v.name}{v.state}{v.protected ? 'yes' : 'no'}{when(v.created)}{shortOcid(v.id)}
No views
+
+ {:else if tab === 'resolvers'} +
+ + + + {#each resolvers as r (r.id)} + + + + + + + {#each endpoints[r.id] ?? [] as e (e.name)} + + {:else} + {#if endpoints[r.id]}{/if} + {/each} + {:else} + + {/each} + +
NameStateVCNDefault viewEndpoints
{r.name}{r.state}{shortOcid(r.vcnId)}{r.defaultViewId ? viewName(r.defaultViewId) : '—'}
{e.name} · {e.kind} · {e.address || 'no address'} · {e.state} · subnet {shortOcid(e.subnetId)}
No endpoints
No resolvers
+
+ {:else if tab === 'tsig'} +

Secrets are never shown in the dashboard.

+
+ + + + {#each tsigKeys as k (k.id)} + + {:else} + + {/each} + +
NameAlgorithmStateCreated
{k.name}{k.algorithm}{k.state}{when(k.created)}
No TSIG keys
+
+ {:else} +
+ + + + {#each policies as p (p.id)} + + {:else} + + {/each} + +
NameTemplateTTLStateCreated
{p.name}{p.template}{p.ttl}{p.state}{when(p.created)}
No steering policies
+
+ {/if} + {/if} +
diff --git a/ui/src/lib/gcpcompute.test.ts b/ui/src/lib/gcpcompute.test.ts index aa66747b5e..3d8741a77d 100644 --- a/ui/src/lib/gcpcompute.test.ts +++ b/ui/src/lib/gcpcompute.test.ts @@ -2,10 +2,30 @@ import { describe, expect, it, vi } from "vitest"; import { COMPUTE_PROXY_BASE, + createInstance, + createSnapshot, deleteAddress, deleteInstance, + imageFamilies, + inRegionScope, + inZoneScope, instanceAction, + instanceTransitioning, labelText, + listImages, + listSecurityPolicies, + listTargetProxies, + metadataRevealable, + operationInScope, + operationRunning, + parseBackendService, + parseForwardingRule, + parseNetwork, + parseSecurityPolicy, + parseUrlMap, + revealMetadata, + shouldPoll, + POLL_MAX, listAddresses, listInstances, listOperations, @@ -26,6 +46,8 @@ const json = (body: unknown, status = 200) => headers: { "Content-Type": "application/json" }, }); const asFetch = (f: unknown) => f as typeof fetch; +const bodyOf = (call: unknown[] | undefined): unknown => + JSON.parse(String(((call ?? [])[1] as RequestInit).body)); describe("parsing", () => { it("derives instance fields from the compute resource", () => { @@ -199,3 +221,287 @@ describe("requests", () => { }); }); }); + +describe("metadata redaction on the client", () => { + it("keeps keys and lengths and never offers SSH keys for reveal", () => { + const i = parseInstance({ + name: "vm", + metadata: { + items: [ + { key: "startup-script", value: "", valueLength: 42, redacted: true }, + { key: "ssh-keys", value: "", valueLength: 300, redacted: true }, + { key: "sshKeys", valueLength: 1 }, + { key: "short", value: "abc" }, + ], + }, + }); + expect(i.metadata).toEqual([ + { key: "startup-script", length: 42, revealable: true }, + { key: "ssh-keys", length: 300, revealable: false }, + { key: "sshKeys", length: 1, revealable: false }, + { key: "short", length: 3, revealable: true }, + ]); + expect(metadataRevealable("block-project-ssh-keys")).toBe(false); + expect(metadataRevealable("")).toBe(false); + }); + + it("requests a reveal only for permitted keys", async () => { + const fetchFn = vi + .fn() + .mockResolvedValue( + json({ metadata: { items: [{ key: "startup-script", value: "echo hi" }] } }), + ); + expect(await revealMetadata("p", "z", "vm", "startup-script", asFetch(fetchFn))).toBe( + "echo hi", + ); + expect(fetchFn.mock.calls[0]?.[0]).toBe( + `${COMPUTE_PROXY_BASE}/projects/p/zones/z/instances/vm?reveal=startup-script`, + ); + expect(await revealMetadata("p", "z", "vm", "ssh-keys", asFetch(fetchFn))).toBe(""); + expect(fetchFn).toHaveBeenCalledTimes(1); + }); +}); + +describe("new resources", () => { + it("reads instance nics, disk modes and tags", () => { + const i = parseInstance({ + name: "vm", + tags: { items: ["web"] }, + disks: [{ source: "x/disks/d", boot: true, mode: "READ_WRITE", interface: "SCSI" }], + networkInterfaces: [ + { + name: "nic0", + network: "x/global/networks/vpc", + subnetwork: "x/regions/r/subnetworks/sub", + networkIP: "10.0.0.2", + accessConfigs: [{ natIP: "34.0.0.1" }], + }, + ], + }); + expect(i.tags).toEqual(["web"]); + expect(i.disks[0]).toMatchObject({ mode: "READ_WRITE", iface: "SCSI" }); + expect(i.nics).toEqual([ + { + name: "nic0", + network: "vpc", + subnetwork: "sub", + internalIp: "10.0.0.2", + externalIps: ["34.0.0.1"], + }, + ]); + }); + + it("reads VPC peering state", () => { + const n = parseNetwork({ + name: "a", + peerings: [ + { name: "p", network: "x/global/networks/b", state: "ACTIVE", exportCustomRoutes: true }, + ], + }); + expect(n.peerings).toEqual([ + { + name: "p", + peer: "b", + state: "ACTIVE", + detail: "", + exportCustomRoutes: true, + importCustomRoutes: false, + }, + ]); + }); + + it("marks subnet routes as system routes", () => { + expect(parseRoute({ name: "default-route-1", routeType: "SUBNET" }).routeType).toBe( + "system (subnet)", + ); + expect(parseRoute({ name: "r" }).routeType).toBe("static"); + }); + + it("parses load balancing resources with the attached Cloud Armor policy", () => { + const b = parseBackendService({ + name: "bs", + region: "x/regions/r", + protocol: "HTTP", + loadBalancingScheme: "EXTERNAL", + backends: [{ group: "x/instanceGroups/ig" }], + healthChecks: ["x/healthChecks/hc"], + securityPolicy: "x/securityPolicies/waf", + }); + expect(b).toMatchObject({ + scope: "r", + backends: ["ig"], + healthChecks: ["hc"], + securityPolicy: "waf", + }); + expect(parseBackendService({ name: "g" }).scope).toBe("global"); + expect( + parseUrlMap({ + name: "um", + defaultService: "x/backendServices/bs", + hostRules: [{ hosts: ["a.com"], pathMatcher: "pm" }], + pathMatchers: [{ name: "pm" }], + }), + ).toMatchObject({ defaultService: "bs", hostRules: ["a.com -> pm"], pathMatchers: ["pm"] }); + expect( + parseForwardingRule({ + name: "fr", + IPAddress: "1.2.3.4", + IPProtocol: "TCP", + portRange: "80-80", + target: "x/targetHttpProxies/tp", + }), + ).toMatchObject({ ip: "1.2.3.4", ports: "80-80", target: "tp" }); + }); + + it("sorts Cloud Armor rules by priority and renders matches", () => { + const p = parseSecurityPolicy({ + name: "waf", + rules: [ + { + priority: 2147483647, + action: "allow", + match: { versionedExpr: "SRC_IPS_V1", config: { srcIpRanges: ["*"] } }, + }, + { + priority: 10, + action: "deny(403)", + preview: true, + match: { expr: { expression: "origin.region_code == 'XX'" } }, + }, + ], + }); + expect(p.rules.map((r) => [r.priority, r.action, r.match, r.preview])).toEqual([ + [10, "deny(403)", "origin.region_code == 'XX'", true], + [2147483647, "allow", "*", false], + ]); + }); + + it("lists images across projects and groups them by family", async () => { + const fetchFn = vi.fn((url: string) => + Promise.resolve( + url.includes("/projects/debian-cloud/") + ? json({ + items: [ + { name: "d12-v1", family: "debian-12" }, + { name: "d12-v2", family: "debian-12" }, + ], + }) + : url.includes("/projects/mine/") + ? json({ items: [{ name: "custom" }] }) + : json({ error: { message: "x" } }, 404), + ), + ); + const images = await listImages("mine", asFetch(fetchFn)); + expect(images.map((i) => `${i.project}/${i.name}`)).toEqual([ + "debian-cloud/d12-v1", + "debian-cloud/d12-v2", + "mine/custom", + ]); + expect(imageFamilies(images)).toEqual([ + { + project: "debian-cloud", + family: "debian-12", + source: "projects/debian-cloud/global/images/family/debian-12", + label: "debian-12 (debian-cloud)", + }, + ]); + }); + + it("merges http and https proxies and security policies", async () => { + const fetchFn = vi.fn((url: string) => + Promise.resolve( + url.includes("targetHttpsProxies") + ? json({ + items: { + global: { + targetHttpsProxies: [{ name: "s", sslCertificates: ["x/sslCertificates/c"] }], + }, + }, + }) + : url.includes("targetHttpProxies") + ? json({ items: { global: { targetHttpProxies: [{ name: "h" }] } } }) + : json({ items: [{ name: "waf" }] }), + ), + ); + const proxies = await listTargetProxies("p", asFetch(fetchFn)); + expect(proxies.map((p) => [p.name, p.kind, p.certificates])).toEqual([ + ["h", "HTTP", []], + ["s", "HTTPS", ["c"]], + ]); + expect((await listSecurityPolicies("p", asFetch(fetchFn)))[0]?.name).toBe("waf"); + }); + + it("creates an instance and a snapshot with fixed-shape bodies", async () => { + const fetchFn = vi.fn((_url: string, _init?: RequestInit) => Promise.resolve(json({}))); + await createInstance( + "p", + "z", + { + name: "vm", + machineType: "e2-small", + sourceImage: "projects/a/global/images/family/f", + diskSizeGb: " 20 ", + network: "vpc", + subnetwork: "", + externalIp: true, + }, + asFetch(fetchFn), + ); + await createSnapshot("p", "z", "disk 1", "snap", asFetch(fetchFn)); + const [first, second] = fetchFn.mock.calls; + expect(first?.[0]).toBe(`${COMPUTE_PROXY_BASE}/projects/p/zones/z/instances`); + expect(bodyOf(first)).toEqual({ + name: "vm", + machineType: "e2-small", + disks: [ + { + boot: true, + autoDelete: true, + initializeParams: { sourceImage: "projects/a/global/images/family/f", diskSizeGb: "20" }, + }, + ], + networkInterfaces: [{ network: "vpc", accessConfigs: [{ type: "ONE_TO_ONE_NAT" }] }], + }); + expect(second?.[0]).toBe( + `${COMPUTE_PROXY_BASE}/projects/p/zones/z/disks/disk%201/createSnapshot`, + ); + expect(bodyOf(second)).toEqual({ name: "snap" }); + }); + + it("supports suspend and resume actions", async () => { + const fetchFn = vi.fn((_url: string, _init?: RequestInit) => Promise.resolve(json({}))); + await instanceAction("p", "z", "vm", "suspend", asFetch(fetchFn)); + await instanceAction("p", "z", "vm", "resume", asFetch(fetchFn)); + expect(fetchFn.mock.calls.map((c) => String(c[0]).split("/").pop())).toEqual([ + "suspend", + "resume", + ]); + }); +}); + +describe("scope and polling", () => { + it("filters zonal, regional and operation scopes", () => { + expect(inZoneScope("us-central1-a", "", "")).toBe(true); + expect(inZoneScope("us-central1-a", "us-central1-a", "us-central1")).toBe(true); + expect(inZoneScope("us-central1-b", "us-central1-a", "us-central1")).toBe(false); + expect(inZoneScope("us-central1-b", "", "us-central1")).toBe(true); + expect(inZoneScope("us-east1-b", "", "us-central1")).toBe(false); + expect(inRegionScope("global", "us-east1")).toBe(true); + expect(inRegionScope("us-west1", "us-east1")).toBe(false); + expect(operationInScope("global", "z-a", "")).toBe(true); + expect(operationInScope("us-central1-a", "", "us-central1")).toBe(true); + expect(operationInScope("us-central1", "us-central1-a", "us-central1")).toBe(true); + expect(operationInScope("europe-west1-b", "", "us-central1")).toBe(false); + }); + + it("polls only while operations or instances are unsettled, and only so long", () => { + expect(operationRunning({ status: "RUNNING" })).toBe(true); + expect(operationRunning({ status: "PENDING" })).toBe(true); + expect(operationRunning({ status: "DONE" })).toBe(false); + expect(instanceTransitioning({ status: "STOPPING" })).toBe(true); + expect(instanceTransitioning({ status: "RUNNING" })).toBe(false); + expect(shouldPoll(true, 0)).toBe(true); + expect(shouldPoll(false, 0)).toBe(false); + expect(shouldPoll(true, POLL_MAX)).toBe(false); + }); +}); diff --git a/ui/src/lib/gcpcompute.ts b/ui/src/lib/gcpcompute.ts index 9f139b305f..500c816ecf 100644 --- a/ui/src/lib/gcpcompute.ts +++ b/ui/src/lib/gcpcompute.ts @@ -5,7 +5,11 @@ import { restCall, shortName, str, strMap, type FetchFn, type Json } from "./gcp export const COMPUTE_PROXY_BASE = "/dashboard/api/v1/gcp/compute"; export const COMPUTE_PROJECT_KEY = "gopherstack_compute_project"; export const COMPUTE_ZONE_KEY = "gopherstack_compute_zone"; +export const COMPUTE_REGION_KEY = "gopherstack_compute_region"; export const ALL_ZONES = ""; +export const ALL_REGIONS = ""; +export const POLL_MS = 2000; +export const POLL_MAX = 45; const DEFAULT_PROJECT = "gopherstack-local"; const enc = encodeURIComponent; @@ -14,9 +18,26 @@ const obj = (v: unknown): Json => (v && typeof v === "object" ? (v as Json) : {} const strs = (v: unknown): string[] => (Array.isArray(v) ? v.map(String) : []); const num = (v: unknown): number => (typeof v === "number" ? v : Number(v) || 0); -export type InstanceAction = "start" | "stop" | "reset"; +export type InstanceAction = "start" | "stop" | "reset" | "suspend" | "resume"; -export type ComputeDiskRef = { name: string; sizeGb: number; boot: boolean }; +export type ComputeDiskRef = { + name: string; + sizeGb: number; + boot: boolean; + mode: string; + iface: string; +}; + +export type InstanceNic = { + name: string; + network: string; + subnetwork: string; + internalIp: string; + externalIps: string[]; +}; + +/** A metadata entry as the proxy returns it: the value is redacted, only its length is known. */ +export type MetadataEntry = { key: string; length: number; revealable: boolean }; export type Instance = { name: string; @@ -28,6 +49,9 @@ export type Instance = { disks: ComputeDiskRef[]; labels: Record; metadataKeys: string[]; + metadata: MetadataEntry[]; + nics: InstanceNic[]; + tags: string[]; created: string; raw: Json; }; @@ -59,10 +83,20 @@ export type Network = { routing: string; mtu: number; subnetworks: string[]; + peerings: Peering[]; created: string; raw: Json; }; +export type Peering = { + name: string; + peer: string; + state: string; + detail: string; + exportCustomRoutes: boolean; + importCustomRoutes: boolean; +}; + export type Subnetwork = { name: string; region: string; @@ -82,11 +116,14 @@ export type Firewall = { ranges: string[]; rules: string[]; targets: string[]; + sourceTags: string[]; raw: Json; }; export type Route = { name: string; + routeType: string; + description: string; network: string; destRange: string; priority: number; @@ -149,6 +186,20 @@ function instanceIps(raw: Json): { internal: string[]; external: string[] } { return { internal, external }; } +/** SSH-related keys are never revealed, in the UI or by the proxy. */ +export function metadataRevealable(key: string): boolean { + return key !== "" && !key.toLowerCase().includes("ssh"); +} + +function parseMetadataEntry(i: Json): MetadataEntry { + const key = str(i.key); + return { + key, + length: typeof i.valueLength === "number" ? i.valueLength : str(i.value).length, + revealable: metadataRevealable(key), + }; +} + export function parseInstance(raw: Json): Instance { const ips = instanceIps(raw); return { @@ -162,9 +213,22 @@ export function parseInstance(raw: Json): Instance { name: shortName(str(d.source)) || str(d.deviceName), sizeGb: num(d.diskSizeGb), boot: d.boot === true, + mode: str(d.mode), + iface: str(d.interface), })), labels: strMap(raw.labels), metadataKeys: list(obj(raw.metadata).items).map((i) => str(i.key)), + metadata: list(obj(raw.metadata).items).map((i) => parseMetadataEntry(i)), + nics: list(raw.networkInterfaces).map((n) => ({ + name: str(n.name), + network: shortName(str(n.network)), + subnetwork: shortName(str(n.subnetwork)), + internalIp: str(n.networkIP), + externalIps: list(n.accessConfigs) + .map((a) => str(a.natIP)) + .filter(Boolean), + })), + tags: strs(obj(raw.tags).items), created: str(raw.creationTimestamp), raw, }; @@ -203,6 +267,14 @@ export function parseNetwork(raw: Json): Network { routing: str(obj(raw.routingConfig).routingMode) || "REGIONAL", mtu: num(raw.mtu), subnetworks: strs(raw.subnetworks), + peerings: list(raw.peerings).map((p) => ({ + name: str(p.name), + peer: shortName(str(p.network)), + state: str(p.state), + detail: str(p.stateDetails), + exportCustomRoutes: p.exportCustomRoutes === true, + importCustomRoutes: p.importCustomRoutes === true, + })), created: str(raw.creationTimestamp), raw, }; @@ -239,6 +311,7 @@ export function parseFirewall(raw: Json): Firewall { ranges: direction === "EGRESS" ? strs(raw.destinationRanges) : strs(raw.sourceRanges), rules: [...allowed, ...denied], targets: strs(raw.targetTags), + sourceTags: strs(raw.sourceTags), raw, }; } @@ -256,6 +329,8 @@ export function parseRoute(raw: Json): Route { const hop = NEXT_HOPS.find((k) => str(raw[k])); return { name: str(raw.name), + routeType: str(raw.routeType) === "SUBNET" ? "system (subnet)" : "static", + description: str(raw.description), network: shortName(str(raw.network)), destRange: str(raw.destRange), priority: num(raw.priority), @@ -498,21 +573,24 @@ export async function deleteAddress( await restCall(COMPUTE_PROXY_BASE, "DELETE", `${base}/${enc(name)}`, undefined, fetchFn); } -export function readStoredComputeScope(): { project: string; zone: string } { +export function readStoredComputeScope(): { project: string; zone: string; region: string } { try { + const zone = localStorage.getItem(COMPUTE_ZONE_KEY)?.trim() || ALL_ZONES; return { project: localStorage.getItem(COMPUTE_PROJECT_KEY)?.trim() || DEFAULT_PROJECT, - zone: localStorage.getItem(COMPUTE_ZONE_KEY)?.trim() || ALL_ZONES, + zone, + region: localStorage.getItem(COMPUTE_REGION_KEY)?.trim() || regionOfZone(zone), }; } catch { - return { project: DEFAULT_PROJECT, zone: ALL_ZONES }; + return { project: DEFAULT_PROJECT, zone: ALL_ZONES, region: ALL_REGIONS }; } } -export function writeStoredComputeScope(project: string, zone: string): void { +export function writeStoredComputeScope(project: string, zone: string, region = ""): void { try { localStorage.setItem(COMPUTE_PROJECT_KEY, project); localStorage.setItem(COMPUTE_ZONE_KEY, zone); + localStorage.setItem(COMPUTE_REGION_KEY, region); } catch { // storage unavailable: the scope applies for this session only } @@ -524,3 +602,397 @@ export function labelText(labels: Record): string { .map(([k, v]) => `${k}=${v}`) .join(", "); } + +export type MachineType = { name: string; vcpus: number; memoryMb: number }; +export type ImageFamily = { project: string; family: string; source: string; label: string }; + +export type NewInstance = { + name: string; + machineType: string; + sourceImage: string; + diskSizeGb: string; + network: string; + subnetwork: string; + externalIp: boolean; +}; + +export type BackendService = { + name: string; + scope: string; + protocol: string; + scheme: string; + backends: string[]; + healthChecks: string[]; + securityPolicy: string; + edgeSecurityPolicy: string; + raw: Json; +}; + +export type UrlMap = { + name: string; + scope: string; + defaultService: string; + hostRules: string[]; + pathMatchers: string[]; + raw: Json; +}; + +export type TargetProxy = { + name: string; + scope: string; + kind: string; + urlMap: string; + certificates: string[]; + raw: Json; +}; + +export type ForwardingRule = { + name: string; + scope: string; + ip: string; + protocol: string; + ports: string; + target: string; + scheme: string; + raw: Json; +}; + +export type SecurityRule = { + priority: number; + action: string; + match: string; + description: string; + preview: boolean; +}; + +export type SecurityPolicy = { + name: string; + type: string; + description: string; + rules: SecurityRule[]; + raw: Json; +}; + +export const IMAGE_PROJECTS = [ + "debian-cloud", + "ubuntu-os-cloud", + "rocky-linux-cloud", + "centos-cloud", +]; + +const scopeName = (raw: Json): string => { + const s = str(raw.zone) || str(raw.region); + return s ? shortName(s) : "global"; +}; + +export function parseMachineType(raw: Json): MachineType { + return { name: str(raw.name), vcpus: num(raw.guestCpus), memoryMb: num(raw.memoryMb) }; +} + +export function parseBackendService(raw: Json): BackendService { + return { + name: str(raw.name), + scope: scopeName(raw), + protocol: str(raw.protocol), + scheme: str(raw.loadBalancingScheme), + backends: list(raw.backends).map((b) => shortName(str(b.group))), + healthChecks: strs(raw.healthChecks).map((h) => shortName(h)), + securityPolicy: shortName(str(raw.securityPolicy)), + edgeSecurityPolicy: shortName(str(raw.edgeSecurityPolicy)), + raw, + }; +} + +export function parseUrlMap(raw: Json): UrlMap { + return { + name: str(raw.name), + scope: scopeName(raw), + defaultService: shortName(str(raw.defaultService)), + hostRules: list(raw.hostRules).map( + (h) => `${strs(h.hosts).join(",")} -> ${str(h.pathMatcher)}`, + ), + pathMatchers: list(raw.pathMatchers).map((m) => str(m.name)), + raw, + }; +} + +export function parseTargetProxy(raw: Json): TargetProxy { + return { + name: str(raw.name), + scope: scopeName(raw), + kind: Array.isArray(raw.sslCertificates) ? "HTTPS" : "HTTP", + urlMap: shortName(str(raw.urlMap)), + certificates: strs(raw.sslCertificates).map((c) => shortName(c)), + raw, + }; +} + +export function parseForwardingRule(raw: Json): ForwardingRule { + const ports = strs(raw.ports); + return { + name: str(raw.name), + scope: scopeName(raw), + ip: str(raw.IPAddress), + protocol: str(raw.IPProtocol), + ports: str(raw.portRange) || ports.join(","), + target: shortName(str(raw.target) || str(raw.backendService)), + scheme: str(raw.loadBalancingScheme), + raw, + }; +} + +function ruleMatch(m: Json): string { + const cfg = obj(m.config); + const ranges = strs(cfg.srcIpRanges); + if (ranges.length > 0) return ranges.join(", "); + const expr = str(obj(m.expr).expression); + return expr || str(m.versionedExpr); +} + +export function parseSecurityPolicy(raw: Json): SecurityPolicy { + return { + name: str(raw.name), + type: str(raw.type) || "CLOUD_ARMOR", + description: str(raw.description), + rules: list(raw.rules) + .map((r) => ({ + priority: num(r.priority), + action: str(r.action), + match: ruleMatch(obj(r.match)), + description: str(r.description), + preview: r.preview === true, + })) + .toSorted((a, b) => a.priority - b.priority), + raw, + }; +} + +/** True while an operation has not reached DONE (drives bounded polling). */ +export function operationRunning(o: Pick): boolean { + return o.status === "PENDING" || o.status === "RUNNING"; +} + +export function instanceTransitioning(i: Pick): boolean { + return ["PROVISIONING", "STAGING", "STOPPING", "SUSPENDING", "REPAIRING"].includes(i.status); +} + +export function validResourceName(name: string): boolean { + return /^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$/.test(name); +} + +export async function listRegionNames( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await listPlain(`${projectPath(project)}/regions`, fetchFn); + return items.map((z) => str(z.name)).toSorted(); +} + +export async function listMachineTypes( + project: string, + zone: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await listPlain(`${zonePath(project, zone)}/machineTypes`, fetchFn); + return items.map((x) => parseMachineType(x)).toSorted((a, b) => a.name.localeCompare(b.name)); +} + +export type ComputeImage = { + name: string; + project: string; + family: string; + status: string; + sizeGb: number; + created: string; + raw: Json; +}; + +export function parseImage(raw: Json, project: string): ComputeImage { + return { + name: str(raw.name), + project, + family: str(raw.family), + status: str(raw.status), + sizeGb: num(raw.diskSizeGb), + created: str(raw.creationTimestamp), + raw, + }; +} + +/** Images of the public image projects plus the project's own; a project that cannot be read is skipped. */ +export async function listImages( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + const projects = [...new Set([...IMAGE_PROJECTS, project])]; + const lists = await Promise.all( + projects.map((p) => + listPlain(`${projectPath(p)}/global/images`, fetchFn).then( + (items) => items.map((x) => parseImage(x, p)), + () => [] as ComputeImage[], + ), + ), + ); + return lists.flat(); +} + +/** One entry per image family. */ +export function imageFamilies(images: ComputeImage[]): ImageFamily[] { + const seen = new Map(); + for (const img of images) { + if (!img.family) continue; + seen.set(`${img.project}/${img.family}`, { + project: img.project, + family: img.family, + source: `projects/${img.project}/global/images/family/${img.family}`, + label: `${img.family} (${img.project})`, + }); + } + return [...seen.values()].toSorted((a, b) => a.label.localeCompare(b.label)); +} + +export async function listBackendServices( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + return (await listAggregated(project, "backendServices", fetchFn)).map((x) => + parseBackendService(x), + ); +} + +export async function listUrlMaps(project: string, fetchFn: FetchFn = fetch): Promise { + return (await listAggregated(project, "urlMaps", fetchFn)).map((x) => parseUrlMap(x)); +} + +export async function listTargetProxies( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + const [http, https] = await Promise.all([ + listAggregated(project, "targetHttpProxies", fetchFn), + listAggregated(project, "targetHttpsProxies", fetchFn), + ]); + return [ + ...http.map((x) => ({ ...parseTargetProxy(x), kind: "HTTP" })), + ...https.map((x) => ({ ...parseTargetProxy(x), kind: "HTTPS" })), + ]; +} + +export async function listForwardingRules( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + return (await listAggregated(project, "forwardingRules", fetchFn)).map((x) => + parseForwardingRule(x), + ); +} + +export async function listSecurityPolicies( + project: string, + fetchFn: FetchFn = fetch, +): Promise { + return (await listPlain(`${projectPath(project)}/global/securityPolicies`, fetchFn)).map((x) => + parseSecurityPolicy(x), + ); +} + +export async function getInstance( + project: string, + zone: string, + name: string, + fetchFn: FetchFn = fetch, +): Promise { + const raw = await restCall( + COMPUTE_PROXY_BASE, + "GET", + `${zonePath(project, zone)}/instances/${enc(name)}`, + undefined, + fetchFn, + ); + return parseInstance(raw); +} + +/** Fetches one metadata value; the proxy refuses SSH-related keys, which come back empty. */ +export async function revealMetadata( + project: string, + zone: string, + name: string, + key: string, + fetchFn: FetchFn = fetch, +): Promise { + if (!metadataRevealable(key)) return ""; + const raw = await restCall( + COMPUTE_PROXY_BASE, + "GET", + `${zonePath(project, zone)}/instances/${enc(name)}?reveal=${enc(key)}`, + undefined, + fetchFn, + ); + const item = list(obj(raw.metadata).items).find((i) => str(i.key) === key); + return str(item?.value); +} + +export async function createInstance( + project: string, + zone: string, + req: NewInstance, + fetchFn: FetchFn = fetch, +): Promise { + const nic: Json = {}; + if (req.network) nic.network = req.network; + if (req.subnetwork) nic.subnetwork = req.subnetwork; + if (req.externalIp) nic.accessConfigs = [{ type: "ONE_TO_ONE_NAT" }]; + const params: Json = { sourceImage: req.sourceImage }; + if (req.diskSizeGb.trim()) params.diskSizeGb = req.diskSizeGb.trim(); + await restCall( + COMPUTE_PROXY_BASE, + "POST", + `${zonePath(project, zone)}/instances`, + { + name: req.name, + machineType: req.machineType, + disks: [{ boot: true, autoDelete: true, initializeParams: params }], + networkInterfaces: [nic], + }, + fetchFn, + ); +} + +export async function createSnapshot( + project: string, + zone: string, + disk: string, + name: string, + fetchFn: FetchFn = fetch, +): Promise { + await restCall( + COMPUTE_PROXY_BASE, + "POST", + `${zonePath(project, zone)}/disks/${enc(disk)}/createSnapshot`, + { name }, + fetchFn, + ); +} + +/** Does a zonal resource belong to the selected zone, or to the selected region when no zone is picked? */ +export function inZoneScope(itemZone: string, zone: string, region: string): boolean { + if (zone) return itemZone === zone; + return !region || regionOfZone(itemZone) === region; +} + +/** Regional resources match their region; global ones are always shown. */ +export function inRegionScope(itemRegion: string, region: string): boolean { + return !region || itemRegion === region || itemRegion === "global"; +} + +/** Keep polling while something is unsettled, for at most POLL_MAX rounds. */ +export function shouldPoll(unsettled: boolean, rounds: number): boolean { + return unsettled && rounds < POLL_MAX; +} + +/** Operations are zonal, regional or global; global ones always show. */ +export function operationInScope(scope: string, zone: string, region: string): boolean { + if (scope === "global" || (!zone && !region)) return true; + if (zone) return scope === zone || scope === regionOfZone(zone); + return scope === region || regionOfZone(scope) === region; +} diff --git a/ui/src/lib/icon-manifest.ts b/ui/src/lib/icon-manifest.ts index b560d0ad53..393bab39a9 100644 --- a/ui/src/lib/icon-manifest.ts +++ b/ui/src/lib/icon-manifest.ts @@ -30,6 +30,7 @@ export const ICON_IDS: ReadonlySet = new Set([ "azure-blob", "azure-containerapps", "azure-cosmosdb", + "azure-entra", "azure-eventgrid", "azure-eventhubs", "azure-file", @@ -37,6 +38,7 @@ export const ICON_IDS: ReadonlySet = new Set([ "azure-functions", "azure-keyvault", "azure-monitor", + "azure-network", "azure-queue", "azure-servicebus", "azure-table", @@ -167,6 +169,8 @@ export const ICON_IDS: ReadonlySet = new Set([ "oci-containerengine", "oci-containerinstances", "oci-core", + "oci-database", + "oci-dns", "oci-events", "oci-functions", "oci-identity", diff --git a/ui/src/lib/ocidatabase.test.ts b/ui/src/lib/ocidatabase.test.ts new file mode 100644 index 0000000000..514f98fb50 --- /dev/null +++ b/ui/src/lib/ocidatabase.test.ts @@ -0,0 +1,327 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + DB_PROXY_BASE, + actionEnabled, + attachmentName, + createBackup, + createBody, + createDatabase, + databaseAction, + deleteDatabase, + downloadWallet, + getDatabase, + getEngine, + isBusy, + listBackups, + listDatabases, + parseAcl, + parseDetail, + parseSummary, + updateBody, + updateDatabase, + validAclEntry, + validateNew, + validatePassword, + type NewDatabase, +} from "./ocidatabase"; + +const json = (body: unknown, status = 200, headers: Record = {}) => + new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json", ...headers }, + }); +const asFetch = (f: unknown) => f as typeof fetch; +const comp = "ocid1.compartment.oc1..aaaa"; + +const dbRaw = { + id: "db1", + displayName: "orders", + dbName: "ORDERS", + dbWorkload: "OLTP", + lifecycleState: "AVAILABLE", + computeModel: "ECPU", + computeCount: 4, + dataStorageSizeInTBs: 2, + isFreeTier: false, + isMtlsConnectionRequired: true, + dbVersion: "23ai", + timeCreated: "2026-11-01T00:00:00Z", + whitelistedIps: ["203.0.113.0/24"], + isAccessControlEnabled: true, + backupRetentionPeriodInDays: 60, + connectionStrings: { + allConnectionStrings: { HIGH: "high-desc" }, + profiles: [ + { + displayName: "orders_high", + consumerGroup: "HIGH", + protocol: "TCPS", + tlsAuthentication: "MUTUAL", + sessionMode: "DIRECT", + value: "high-desc", + }, + ], + }, + connectionUrls: { apexUrl: "https://apex.example", ignored: 5 }, +}; + +const newDb: NewDatabase = { + compartmentId: comp, + displayName: " orders ", + dbName: "ORDERS", + workload: "OLTP", + computeCount: 2, + storageTBs: 1, + freeTier: false, + mtlsRequired: true, + adminPassword: "Welcome12345!x", +}; + +describe("parsing", () => { + it("parses a summary", () => { + expect(parseSummary(dbRaw)).toEqual({ + id: "db1", + name: "orders", + dbName: "ORDERS", + workload: "OLTP", + state: "AVAILABLE", + details: "", + computeModel: "ECPU", + computeCount: 4, + storageTBs: 2, + freeTier: false, + mtlsRequired: true, + version: "23ai", + created: "2026-11-01T00:00:00Z", + }); + expect(parseSummary({ dbName: "X" }).name).toBe("X"); + }); + + it("parses connection profiles, urls and access control", () => { + const d = parseDetail(dbRaw); + expect(d.profiles).toEqual([ + { + name: "orders_high", + consumerGroup: "HIGH", + protocol: "TCPS", + tlsAuthentication: "MUTUAL", + sessionMode: "DIRECT", + value: "high-desc", + }, + ]); + expect(d.connectionStrings).toEqual({ HIGH: "high-desc" }); + expect(d.connectionUrls).toEqual({ apexUrl: "https://apex.example" }); + expect(d).toMatchObject({ acl: ["203.0.113.0/24"], aclEnabled: true, backupRetentionDays: 60 }); + expect(parseDetail({ id: "x" }).profiles).toEqual([]); + }); +}); + +describe("state helpers", () => { + it("offers actions by state", () => { + expect(actionEnabled("stop", "AVAILABLE")).toBe(true); + expect(actionEnabled("restart", "AVAILABLE")).toBe(true); + expect(actionEnabled("start", "AVAILABLE")).toBe(false); + expect(actionEnabled("start", "STOPPED")).toBe(true); + expect(actionEnabled("stop", "STOPPED")).toBe(false); + expect(actionEnabled("stop", "PROVISIONING")).toBe(false); + }); + + it("detects busy states", () => { + expect(isBusy("PROVISIONING")).toBe(true); + expect(isBusy("STOPPING")).toBe(true); + expect(isBusy("AVAILABLE")).toBe(false); + expect(isBusy("STOPPED")).toBe(false); + }); +}); + +describe("create form", () => { + it("builds the body from fixed fields", () => { + expect(createBody(newDb)).toEqual({ + compartmentId: comp, + dbName: "ORDERS", + dbWorkload: "OLTP", + adminPassword: "Welcome12345!x", + isMtlsConnectionRequired: true, + isFreeTier: false, + displayName: "orders", + computeModel: "ECPU", + computeCount: 2, + dataStorageSizeInTBs: 1, + }); + }); + + it("leaves the shape out for Always Free", () => { + const body = createBody({ ...newDb, freeTier: true, displayName: "" }); + expect(body).not.toHaveProperty("computeCount"); + expect(body).not.toHaveProperty("displayName"); + expect(body.isFreeTier).toBe(true); + }); + + it("validates fields", () => { + expect(validateNew(newDb)).toBe(""); + expect(validateNew({ ...newDb, dbName: "1bad" })).toMatch(/Database name/); + expect(validateNew({ ...newDb, dbName: "has space" })).toMatch(/Database name/); + expect(validateNew({ ...newDb, workload: "XX" })).toMatch(/workload/); + expect(validateNew({ ...newDb, computeCount: 1 })).toMatch(/ECPU/); + expect(validateNew({ ...newDb, storageTBs: 0 })).toMatch(/Storage/); + expect(validateNew({ ...newDb, freeTier: true, computeCount: 0, storageTBs: 0 })).toBe(""); + }); + + it("checks the admin password policy", () => { + expect(validatePassword("Welcome12345")).toBe(""); + expect(validatePassword("short1A")).toMatch(/12 to 30/); + expect(validatePassword("alllowercase123")).toMatch(/uppercase/); + expect(validatePassword('Welcome1234"5')).toMatch(/cannot contain/); + }); + + it("sends the body through the proxy", async () => { + const fetchMock = vi.fn(() => Promise.resolve(json({}, 202))); + await createDatabase(newDb, asFetch(fetchMock)); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe(`${DB_PROXY_BASE}/api/autonomousDatabases`); + expect(init.method).toBe("POST"); + }); +}); + +describe("requests", () => { + it("lists databases, hiding terminated ones", async () => { + const fetchMock = vi.fn((_url: string) => + Promise.resolve(json([dbRaw, { ...dbRaw, id: "db2", lifecycleState: "TERMINATED" }])), + ); + const got = await listDatabases(comp, asFetch(fetchMock)); + expect(got.map((d) => d.id)).toEqual(["db1"]); + expect(fetchMock.mock.calls[0][0]).toBe( + `${DB_PROXY_BASE}/api/autonomousDatabases?compartmentId=${encodeURIComponent(comp)}`, + ); + }); + + it("gets one database and the engine", async () => { + expect( + (await getDatabase("db 1", asFetch(vi.fn(() => Promise.resolve(json(dbRaw)))))).dbName, + ).toBe("ORDERS"); + expect(await getEngine(asFetch(vi.fn(() => Promise.resolve(json({ engine: "stub" })))))).toBe( + "stub", + ); + }); + + it("runs actions and delete", async () => { + const fetchMock = vi.fn(() => Promise.resolve(json({}))); + await databaseAction("db1", "stop", asFetch(fetchMock)); + expect(fetchMock).toHaveBeenLastCalledWith( + `${DB_PROXY_BASE}/api/autonomousDatabases/db1/actions/stop`, + expect.objectContaining({ method: "POST" }), + ); + await deleteDatabase("db1", asFetch(fetchMock)); + expect(fetchMock).toHaveBeenLastCalledWith( + `${DB_PROXY_BASE}/api/autonomousDatabases/db1`, + expect.objectContaining({ method: "DELETE" }), + ); + }); + + it("builds update bodies from the fields set", async () => { + expect(updateBody({ displayName: " x " })).toEqual({ displayName: "x" }); + expect(updateBody({ acl: ["1.2.3.4"], aclEnabled: true, computeCount: 4 })).toEqual({ + whitelistedIps: ["1.2.3.4"], + isAccessControlEnabled: true, + computeCount: 4, + }); + const fetchMock = vi.fn(() => Promise.resolve(json({}))); + await updateDatabase("db1", { computeCount: 8 }, asFetch(fetchMock)); + expect((fetchMock.mock.calls[0] as unknown as [string, RequestInit])[1].method).toBe("PUT"); + }); + + it("parses the access control list", () => { + expect(parseAcl("1.1.1.1, 2.2.2.0/24\n3.3.3.3\n")).toEqual([ + "1.1.1.1", + "2.2.2.0/24", + "3.3.3.3", + ]); + expect(parseAcl("")).toEqual([]); + expect(validAclEntry("2001:db8::/32")).toBe(true); + expect(validAclEntry("bad entry")).toBe(false); + }); + + it("lists backups newest first without deleted ones", async () => { + const fetchMock = vi.fn((_url: string) => + Promise.resolve( + json([ + { + id: "b1", + displayName: "old", + timeStarted: "2026-10-01T00:00:00Z", + lifecycleState: "ACTIVE", + }, + { + id: "b2", + displayName: "new", + timeStarted: "2026-11-01T00:00:00Z", + lifecycleState: "CREATING", + isAutomatic: true, + }, + { + id: "b3", + displayName: "gone", + timeStarted: "2026-11-02T00:00:00Z", + lifecycleState: "DELETED", + }, + ]), + ), + ); + const got = await listBackups(comp, "db1", asFetch(fetchMock)); + expect(got.map((b) => b.id)).toEqual(["b2", "b1"]); + expect(got[0].automatic).toBe(true); + expect(fetchMock.mock.calls[0][0]).toContain("autonomousDatabaseId=db1"); + }); + + it("creates a manual backup", async () => { + const fetchMock = vi.fn(() => Promise.resolve(json({}, 202))); + await createBackup("db1", " nightly ", asFetch(fetchMock)); + const init = (fetchMock.mock.calls[0] as unknown as [string, RequestInit])[1]; + expect(JSON.parse(init.body as string)).toEqual({ + autonomousDatabaseId: "db1", + displayName: "nightly", + }); + }); +}); + +describe("wallet", () => { + it("reads the attachment file name", () => { + expect(attachmentName('attachment; filename="Wallet_ORDERS.zip"', "x.zip")).toBe( + "Wallet_ORDERS.zip", + ); + expect(attachmentName(null, "wallet.zip")).toBe("wallet.zip"); + expect(attachmentName('attachment; filename="../evil"', "wallet.zip")).toBe("wallet.zip"); + }); + + it("posts the password and returns the zip with its name", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + new Response(new Uint8Array([80, 75, 3, 4]), { + status: 200, + headers: { + "Content-Type": "application/zip", + "Content-Disposition": 'attachment; filename="Wallet_ORDERS.zip"', + }, + }), + ), + ); + const got = await downloadWallet("db1", "Wallet#1234", asFetch(fetchMock)); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe(`${DB_PROXY_BASE}/api/autonomousDatabases/db1/wallet`); + expect(JSON.parse(init.body as string)).toEqual({ password: "Wallet#1234" }); + expect(got.filename).toBe("Wallet_ORDERS.zip"); + expect(got.blob.size).toBe(4); + }); + + it("raises the service error without echoing the password", async () => { + const fetchMock = vi.fn(() => + Promise.resolve(json({ code: "InvalidParameter", message: "password too weak" }, 400)), + ); + const err = await downloadWallet("db1", "Wallet#1234", asFetch(fetchMock)).catch( + (e: unknown) => e, + ); + expect((err as Error).message).toBe("password too weak"); + expect(JSON.stringify(err)).not.toContain("Wallet#1234"); + }); +}); diff --git a/ui/src/lib/ocidatabase.ts b/ui/src/lib/ocidatabase.ts new file mode 100644 index 0000000000..d080663dfe --- /dev/null +++ b/ui/src/lib/ocidatabase.ts @@ -0,0 +1,378 @@ +/** Client for the Oracle Cloud Autonomous Database emulator through the dashboard's in-process proxy. */ + +import { + ociCompartments, + ociGetOne, + ociListPages, + readStoredValue, + writeStoredValue, +} from "./ocirest.ts"; +import { GcpRestError, restCall, str, type FetchFn, type Json } from "./gcprest.ts"; + +export const DB_PROXY_BASE = "/dashboard/api/v1/oci/database"; +export const DB_COMPARTMENT_KEY = "gopherstack_ocidb_compartment"; + +export const WORKLOADS = ["OLTP", "DW", "AJD", "APEX"] as const; + +const enc = encodeURIComponent; +const num = (v: unknown): number => (typeof v === "number" ? v : Number(v) || 0); +const obj = (v: unknown): Json => (v && typeof v === "object" ? (v as Json) : {}); +const list = (v: unknown): Json[] => (Array.isArray(v) ? (v as Json[]) : []); +const strs = (v: unknown): string[] => (Array.isArray(v) ? v.map(String) : []); + +export type DbSummary = { + id: string; + name: string; + dbName: string; + workload: string; + state: string; + details: string; + computeModel: string; + computeCount: number; + storageTBs: number; + freeTier: boolean; + mtlsRequired: boolean; + version: string; + created: string; +}; + +export type ConnectionProfile = { + name: string; + consumerGroup: string; + protocol: string; + tlsAuthentication: string; + sessionMode: string; + value: string; +}; + +export type DbDetail = DbSummary & { + profiles: ConnectionProfile[]; + connectionStrings: Record; + connectionUrls: Record; + acl: string[]; + aclEnabled: boolean; + privateEndpoint: string; + backupRetentionDays: number; +}; + +export type DbBackup = { + id: string; + name: string; + type: string; + state: string; + automatic: boolean; + started: string; + ended: string; + availableTill: string; +}; + +export function parseSummary(raw: Json): DbSummary { + return { + id: str(raw.id), + name: str(raw.displayName) || str(raw.dbName), + dbName: str(raw.dbName), + workload: str(raw.dbWorkload), + state: str(raw.lifecycleState), + details: str(raw.lifecycleDetails), + computeModel: str(raw.computeModel), + computeCount: num(raw.computeCount), + storageTBs: num(raw.dataStorageSizeInTBs), + freeTier: raw.isFreeTier === true, + mtlsRequired: raw.isMtlsConnectionRequired === true, + version: str(raw.dbVersion), + created: str(raw.timeCreated), + }; +} + +function stringMap(v: unknown): Record { + const out: Record = {}; + for (const [k, val] of Object.entries(obj(v))) if (typeof val === "string") out[k] = val; + return out; +} + +export function parseDetail(raw: Json): DbDetail { + const cs = obj(raw.connectionStrings); + return { + ...parseSummary(raw), + profiles: list(cs.profiles).map((p) => ({ + name: str(p.displayName), + consumerGroup: str(p.consumerGroup), + protocol: str(p.protocol), + tlsAuthentication: str(p.tlsAuthentication), + sessionMode: str(p.sessionMode), + value: str(p.value), + })), + connectionStrings: stringMap(cs.allConnectionStrings), + connectionUrls: stringMap(raw.connectionUrls), + acl: strs(raw.whitelistedIps), + aclEnabled: raw.isAccessControlEnabled === true, + privateEndpoint: str(raw.privateEndpoint), + backupRetentionDays: num(raw.backupRetentionPeriodInDays), + }; +} + +export function parseBackup(raw: Json): DbBackup { + return { + id: str(raw.id), + name: str(raw.displayName), + type: str(raw.type), + state: str(raw.lifecycleState), + automatic: raw.isAutomatic === true, + started: str(raw.timeStarted), + ended: str(raw.timeEnded), + availableTill: str(raw.timeAvailableTill), + }; +} + +export const listCompartments = (fetchFn: FetchFn = fetch) => + ociCompartments(DB_PROXY_BASE, fetchFn); + +export async function getEngine(fetchFn: FetchFn = fetch): Promise { + return str((await restCall(DB_PROXY_BASE, "GET", "/engine", undefined, fetchFn)).engine); +} + +const compQuery = (compartment: string) => `compartmentId=${enc(compartment)}`; + +export async function listDatabases( + compartment: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DB_PROXY_BASE, + "/api/autonomousDatabases", + compQuery(compartment), + "", + fetchFn, + ); + return items.map((d) => parseSummary(d)).filter((d) => d.state !== "TERMINATED"); +} + +export async function getDatabase(id: string, fetchFn: FetchFn = fetch): Promise { + return parseDetail( + await ociGetOne(DB_PROXY_BASE, `/api/autonomousDatabases/${enc(id)}`, fetchFn), + ); +} + +export async function listBackups( + compartment: string, + databaseId: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DB_PROXY_BASE, + "/api/autonomousDatabaseBackups", + `${compQuery(compartment)}&autonomousDatabaseId=${enc(databaseId)}`, + "", + fetchFn, + ); + return items + .map((b) => parseBackup(b)) + .filter((b) => b.state !== "DELETED") + .toSorted((a, b) => b.started.localeCompare(a.started)); +} + +export type DbAction = "start" | "stop" | "restart"; + +/** Start, stop and restart are only offered from the states the service accepts them in. */ +export function actionEnabled(action: DbAction, state: string): boolean { + return action === "start" ? state === "STOPPED" : state === "AVAILABLE"; +} + +const BUSY = new Set([ + "PROVISIONING", + "STARTING", + "STOPPING", + "RESTARTING", + "UPDATING", + "SCALE_IN_PROGRESS", + "RESTORE_IN_PROGRESS", + "BACKUP_IN_PROGRESS", + "TERMINATING", +]); + +export const isBusy = (state: string): boolean => BUSY.has(state); + +export async function databaseAction( + id: string, + action: DbAction, + fetchFn: FetchFn = fetch, +): Promise { + await restCall( + DB_PROXY_BASE, + "POST", + `/api/autonomousDatabases/${enc(id)}/actions/${action}`, + undefined, + fetchFn, + ); +} + +export async function deleteDatabase(id: string, fetchFn: FetchFn = fetch): Promise { + await restCall( + DB_PROXY_BASE, + "DELETE", + `/api/autonomousDatabases/${enc(id)}`, + undefined, + fetchFn, + ); +} + +export type DbUpdate = { + displayName?: string; + acl?: string[]; + aclEnabled?: boolean; + computeCount?: number; +}; + +export function updateBody(u: DbUpdate): Json { + const body: Json = {}; + if (u.displayName !== undefined) body.displayName = u.displayName.trim(); + if (u.acl !== undefined) body.whitelistedIps = u.acl; + if (u.aclEnabled !== undefined) body.isAccessControlEnabled = u.aclEnabled; + if (u.computeCount !== undefined) body.computeCount = u.computeCount; + return body; +} + +export async function updateDatabase( + id: string, + u: DbUpdate, + fetchFn: FetchFn = fetch, +): Promise { + await restCall( + DB_PROXY_BASE, + "PUT", + `/api/autonomousDatabases/${enc(id)}`, + updateBody(u), + fetchFn, + ); +} + +/** Splits an access control list typed one entry per line or comma separated. */ +export const parseAcl = (text: string): string[] => + text + .split(/[\s,]+/) + .map((s) => s.trim()) + .filter((s) => s !== ""); + +export const validAclEntry = (s: string): boolean => /^[A-Za-z0-9.:/_-]{1,255}$/.test(s); + +export type NewDatabase = { + compartmentId: string; + displayName: string; + dbName: string; + workload: string; + computeCount: number; + storageTBs: number; + freeTier: boolean; + mtlsRequired: boolean; + adminPassword: string; +}; + +export function validatePassword(pw: string): string { + if (pw.length < 12 || pw.length > 30) return "Admin password must be 12 to 30 characters"; + if (!/[A-Z]/.test(pw) || !/[a-z]/.test(pw) || !/[0-9]/.test(pw)) { + return "Admin password needs an uppercase letter, a lowercase letter and a digit"; + } + if (pw.includes('"')) return 'Admin password cannot contain "'; + return ""; +} + +export function validateNew(n: NewDatabase): string { + if (!/^[A-Za-z][A-Za-z0-9]{0,29}$/.test(n.dbName)) { + return "Database name must start with a letter and use letters and digits only (30 max)"; + } + if (!(WORKLOADS as readonly string[]).includes(n.workload)) return "Pick a workload"; + if (!n.freeTier) { + if (!(n.computeCount >= 2)) return "Compute must be at least 2 ECPUs"; + if (!Number.isInteger(n.storageTBs) || n.storageTBs < 1) return "Storage must be at least 1 TB"; + } + return validatePassword(n.adminPassword); +} + +export function createBody(n: NewDatabase): Json { + const body: Json = { + compartmentId: n.compartmentId, + dbName: n.dbName.trim(), + dbWorkload: n.workload, + adminPassword: n.adminPassword, + isMtlsConnectionRequired: n.mtlsRequired, + isFreeTier: n.freeTier, + }; + if (n.displayName.trim() !== "") body.displayName = n.displayName.trim(); + if (!n.freeTier) { + body.computeModel = "ECPU"; + body.computeCount = n.computeCount; + body.dataStorageSizeInTBs = n.storageTBs; + } + return body; +} + +export async function createDatabase(n: NewDatabase, fetchFn: FetchFn = fetch): Promise { + await restCall(DB_PROXY_BASE, "POST", "/api/autonomousDatabases", createBody(n), fetchFn); +} + +export async function createBackup( + databaseId: string, + displayName: string, + fetchFn: FetchFn = fetch, +): Promise { + const body: Json = { autonomousDatabaseId: databaseId }; + if (displayName.trim() !== "") body.displayName = displayName.trim(); + await restCall(DB_PROXY_BASE, "POST", "/api/autonomousDatabaseBackups", body, fetchFn); +} + +export type WalletFile = { blob: Blob; filename: string }; + +/** Pulls the file name out of a Content-Disposition attachment header. */ +export function attachmentName(header: string | null, fallback: string): string { + const m = /filename="([^"\\/]+)"/.exec(header ?? ""); + return m?.[1] ?? fallback; +} + +/** Wallet zip for a database. The password only travels in this request body. */ +export async function downloadWallet( + id: string, + password: string, + fetchFn: FetchFn = fetch, +): Promise { + let res: Response; + try { + res = await fetchFn(`${DB_PROXY_BASE}/api/autonomousDatabases/${enc(id)}/wallet`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password }), + }); + } catch (e) { + throw new GcpRestError(0, "NetworkError", e instanceof Error ? e.message : String(e)); + } + if (!res.ok) { + let body: Json = {}; + try { + body = obj(JSON.parse(await res.text())); + } catch { + // non-JSON error body: fall back to the status text + } + const proxy = obj(body.error); + throw new GcpRestError( + res.status, + str(proxy.status) || str(body.code), + str(proxy.message) || str(body.message) || res.statusText || `HTTP ${res.status}`, + ); + } + return { + blob: await res.blob(), + filename: attachmentName(res.headers.get("Content-Disposition"), "wallet.zip"), + }; +} + +/** Hands a downloaded file to the browser through a temporary object URL. */ +export function saveBlob(file: WalletFile): void { + const url = URL.createObjectURL(file.blob); + const a = document.createElement("a"); + a.href = url; + a.download = file.filename; + a.click(); + URL.revokeObjectURL(url); +} + +export const readStoredDbCompartment = () => readStoredValue(DB_COMPARTMENT_KEY); +export const writeStoredDbCompartment = (id: string) => writeStoredValue(DB_COMPARTMENT_KEY, id); diff --git a/ui/src/lib/ocidns.test.ts b/ui/src/lib/ocidns.test.ts new file mode 100644 index 0000000000..b9715aed45 --- /dev/null +++ b/ui/src/lib/ocidns.test.ts @@ -0,0 +1,335 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + DNS_PROXY_BASE, + addOp, + createZone, + createZoneBody, + deleteZone, + editOps, + getDataPlane, + getZoneRecords, + groupRecords, + isConflict, + listEndpoints, + listResolvers, + listSteeringPolicies, + listTsigKeys, + listViews, + listZones, + parseRecord, + parseZone, + patchRecords, + removeOp, + tryQuery, + validateQuery, + validateRecord, + validateZone, +} from "./ocidns"; +import { GcpRestError } from "./gcprest"; + +const json = (body: unknown, status = 200, headers: Record = {}) => + new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json", ...headers }, + }); +const asFetch = (f: unknown) => f as typeof fetch; +const comp = "ocid1.compartment.oc1..aaaa"; + +const zoneRaw = { + id: "z1", + name: "example.com", + zoneType: "PRIMARY", + scope: "GLOBAL", + serial: 7, + timeCreated: "2026-11-01T00:00:00Z", + lifecycleState: "ACTIVE", + nameservers: [{ hostname: "ns1.example.net" }], +}; + +const rec = (domain: string, rtype: string, rdata: string, ttl = 300) => ({ + domain, + rtype, + rdata, + ttl, + isProtected: false, +}); + +describe("parsing", () => { + it("parses a zone", () => { + expect(parseZone(zoneRaw)).toEqual({ + id: "z1", + name: "example.com", + type: "PRIMARY", + scope: "GLOBAL", + serial: 7, + created: "2026-11-01T00:00:00Z", + state: "ACTIVE", + viewId: "", + nameservers: ["ns1.example.net"], + }); + }); + + it("parses a record and groups by rrset", () => { + expect( + parseRecord({ ...rec("a.example.com", "A", "1.1.1.1"), isProtected: true }).protected, + ).toBe(true); + const sets = groupRecords([ + parseRecord(rec("b.example.com", "A", "2.2.2.2")), + parseRecord(rec("a.example.com", "TXT", "hi")), + parseRecord(rec("a.example.com", "A", "1.1.1.2")), + parseRecord(rec("a.example.com", "A", "1.1.1.1")), + ]); + expect(sets.map((s) => `${s.domain} ${s.rtype} ${s.records.length}`)).toEqual([ + "a.example.com A 2", + "a.example.com TXT 1", + "b.example.com A 1", + ]); + }); +}); + +describe("record operations", () => { + const draft = { domain: " www.example.com. ", rtype: "A", ttl: 60, rdata: " 192.0.2.1 " }; + + it("builds ADD and REMOVE from fixed fields", () => { + const want = { domain: "www.example.com", rtype: "A", ttl: 60, rdata: "192.0.2.1" }; + expect(addOp(draft)).toEqual({ operation: "ADD", ...want }); + expect(removeOp(draft)).toEqual({ operation: "REMOVE", ...want }); + }); + + it("edits as remove then add", () => { + const ops = editOps( + { domain: "www.example.com", rtype: "A", ttl: 60, rdata: "192.0.2.1" }, + { domain: "www.example.com", rtype: "A", ttl: 120, rdata: "192.0.2.2" }, + ); + expect(ops.map((o) => [o.operation, o.rdata, o.ttl])).toEqual([ + ["REMOVE", "192.0.2.1", 60], + ["ADD", "192.0.2.2", 120], + ]); + }); + + it("validates a record against its zone", () => { + const ok = { domain: "www.example.com", rtype: "A", ttl: 300, rdata: "192.0.2.1" }; + expect(validateRecord("example.com", ok)).toBe(""); + expect(validateRecord("example.com", { ...ok, domain: "example.com" })).toBe(""); + expect(validateRecord("example.com", { ...ok, domain: "" })).toMatch(/required/); + expect(validateRecord("example.com", { ...ok, domain: "evil.org" })).toMatch(/must be/); + expect(validateRecord("example.com", { ...ok, domain: "notexample.com" })).toMatch(/must be/); + expect(validateRecord("example.com", { ...ok, domain: "a b.example.com" })).toMatch(/invalid/); + expect(validateRecord("example.com", { ...ok, rtype: "SOA" })).toMatch(/type/); + expect(validateRecord("example.com", { ...ok, rdata: " " })).toMatch(/data/); + expect(validateRecord("example.com", { ...ok, ttl: -1 })).toMatch(/TTL/); + expect(validateRecord("example.com", { ...ok, ttl: 1.5 })).toMatch(/TTL/); + }); + + it("recognises precondition failures as conflicts", () => { + expect(isConflict(new GcpRestError(412, "PreconditionFailed", "x"))).toBe(true); + expect(isConflict(new GcpRestError(409, "Conflict", "x"))).toBe(true); + expect(isConflict(new GcpRestError(404, "NotFound", "x"))).toBe(false); + expect(isConflict(new Error("x"))).toBe(false); + }); +}); + +describe("zones", () => { + it("lists both scopes sorted by name and hides deleted zones", async () => { + const fetchMock = vi.fn((url: string) => + Promise.resolve( + url.includes("scope=PRIVATE") + ? json([{ ...zoneRaw, id: "z2", name: "a.internal", scope: "PRIVATE", viewId: "v1" }]) + : json([zoneRaw, { ...zoneRaw, id: "z3", name: "gone.com", lifecycleState: "DELETED" }]), + ), + ); + const zones = await listZones(comp, asFetch(fetchMock)); + expect(zones.map((z) => z.name)).toEqual(["a.internal", "example.com"]); + const urls = fetchMock.mock.calls.map((c) => c[0] as string); + expect(urls).toContain( + `${DNS_PROXY_BASE}/api/zones?compartmentId=${encodeURIComponent(comp)}&scope=GLOBAL`, + ); + expect(urls.some((u) => u.endsWith("&scope=PRIVATE"))).toBe(true); + }); + + it("builds the create body and validates it", () => { + const base = { + name: " example.com ", + compartmentId: comp, + scope: "GLOBAL" as const, + viewId: "v1", + }; + expect(createZoneBody(base)).toEqual({ + name: "example.com", + compartmentId: comp, + zoneType: "PRIMARY", + scope: "GLOBAL", + }); + expect(createZoneBody({ ...base, scope: "PRIVATE" })).toMatchObject({ + scope: "PRIVATE", + viewId: "v1", + }); + expect(validateZone(base)).toBe(""); + expect(validateZone({ ...base, name: "" })).toMatch(/required/); + expect(validateZone({ ...base, name: "nodots" })).toMatch(/domain/); + expect(validateZone({ ...base, scope: "PRIVATE", viewId: "" })).toMatch(/view/); + }); + + it("creates and deletes zones through the proxy", async () => { + const fetchMock = vi.fn(() => Promise.resolve(json({}))); + await createZone( + { name: "example.com", compartmentId: comp, scope: "GLOBAL", viewId: "" }, + asFetch(fetchMock), + ); + expect(fetchMock).toHaveBeenLastCalledWith( + `${DNS_PROXY_BASE}/api/zones`, + expect.objectContaining({ method: "POST" }), + ); + await deleteZone({ id: "z2", scope: "PRIVATE", viewId: "v 1" }, asFetch(fetchMock)); + expect(fetchMock).toHaveBeenLastCalledWith( + `${DNS_PROXY_BASE}/api/zones/z2?scope=PRIVATE&viewId=v%201`, + expect.objectContaining({ method: "DELETE" }), + ); + }); +}); + +describe("records", () => { + it("collects records over pages and keeps the first etag", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + json({ items: [rec("a.example.com", "A", "1.1.1.1")] }, 200, { + etag: "e1", + "opc-next-page": "p2", + }), + ) + .mockResolvedValueOnce( + json({ items: [rec("b.example.com", "A", "2.2.2.2")] }, 200, { etag: "e2" }), + ); + const got = await getZoneRecords({ id: "z1", scope: "GLOBAL", viewId: "" }, asFetch(fetchMock)); + expect(got.etag).toBe("e1"); + expect(got.records.map((r) => r.domain)).toEqual(["a.example.com", "b.example.com"]); + expect(fetchMock.mock.calls[1][0]).toBe(`${DNS_PROXY_BASE}/api/zones/z1/records?page=p2`); + }); + + it("sends the etag as If-Match with the operations", async () => { + const fetchMock = vi.fn(() => Promise.resolve(json({}))); + const ops = [addOp({ domain: "www.example.com", rtype: "A", ttl: 60, rdata: "192.0.2.1" })]; + await patchRecords({ id: "z1", scope: "GLOBAL", viewId: "" }, ops, "e1", asFetch(fetchMock)); + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]; + expect(url).toBe(`${DNS_PROXY_BASE}/api/zones/z1/records`); + expect(init.method).toBe("PATCH"); + expect((init.headers as Record)["If-Match"]).toBe("e1"); + expect(JSON.parse(init.body as string)).toEqual({ items: ops }); + }); + + it("surfaces a precondition failure as a conflict", async () => { + const fetchMock = vi.fn(() => + Promise.resolve(json({ code: "PreconditionFailed", message: "etag mismatch" }, 412)), + ); + const err = await patchRecords( + { id: "z1", scope: "GLOBAL", viewId: "" }, + [], + "stale", + asFetch(fetchMock), + ).catch((e: unknown) => e); + expect(isConflict(err)).toBe(true); + expect((err as Error).message).toBe("etag mismatch"); + }); +}); + +const reply = (body: unknown) => asFetch(vi.fn(() => Promise.resolve(json(body)))); + +describe("read-only lists", () => { + it("parses views, resolvers, endpoints, TSIG keys and policies", async () => { + expect( + await listViews( + comp, + reply([{ id: "v1", displayName: "main", isProtected: true, lifecycleState: "ACTIVE" }]), + ), + ).toEqual([{ id: "v1", name: "main", created: "", protected: true, state: "ACTIVE" }]); + expect( + ( + await listResolvers( + comp, + reply([{ id: "r1", displayName: "res", attachedVcnId: "vcn", defaultViewId: "v1" }]), + ) + )[0], + ).toMatchObject({ + id: "r1", + name: "res", + vcnId: "vcn", + }); + const eps = await listEndpoints( + "r1", + reply([ + { name: "fwd", isForwarding: true, forwardingAddress: "10.0.0.5", subnetId: "s1" }, + { name: "lst", isListening: true, listeningAddress: "10.0.0.6" }, + ]), + ); + expect(eps.map((e) => [e.name, e.kind, e.address])).toEqual([ + ["fwd", "Forwarding", "10.0.0.5"], + ["lst", "Listening", "10.0.0.6"], + ]); + const keys = await listTsigKeys( + comp, + reply([{ id: "k1", name: "key", algorithm: "hmac-sha256", secret: "LEAK" }]), + ); + expect(keys).toEqual([ + { id: "k1", name: "key", algorithm: "hmac-sha256", state: "", created: "" }, + ]); + expect(JSON.stringify(keys)).not.toContain("LEAK"); + expect( + ( + await listSteeringPolicies( + comp, + reply([{ id: "p1", displayName: "pol", template: "FAILOVER", ttl: 30 }]), + ) + )[0], + ).toMatchObject({ + name: "pol", + template: "FAILOVER", + ttl: 30, + }); + }); +}); + +describe("try it", () => { + it("validates names and types", () => { + expect(validateQuery("www.example.com", "A")).toBe(""); + expect(validateQuery("www.example.com.", "SOA")).toBe(""); + expect(validateQuery("", "A")).toMatch(/Enter/); + expect(validateQuery("bad name.com", "A")).toMatch(/valid/); + expect(validateQuery("a..com", "A")).toMatch(/valid/); + expect(validateQuery(`${"a".repeat(64)}.com`, "A")).toMatch(/valid/); + expect(validateQuery("example.com", "AXFR")).toMatch(/type/); + }); + + it("reports the data plane state", async () => { + const f = asFetch(vi.fn(() => Promise.resolve(json({ enabled: true, port: 8169 })))); + expect(await getDataPlane(f)).toEqual({ enabled: true, port: 8169 }); + }); + + it("posts the query and parses the answer section", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + json({ + rcode: "NOERROR", + authoritative: true, + name: "www.example.com.", + type: "A", + server: "127.0.0.1:8169", + answer: [{ name: "www.example.com.", type: "A", ttl: 60, data: "192.0.2.1" }], + authority: [], + }), + ), + ); + const got = await tryQuery(" www.example.com ", "A", asFetch(fetchMock)); + expect( + JSON.parse((fetchMock.mock.calls[0] as unknown as [string, RequestInit])[1].body as string), + ).toEqual({ + name: "www.example.com", + type: "A", + }); + expect(got).toMatchObject({ rcode: "NOERROR", authoritative: true }); + expect(got.answer).toEqual([ + { name: "www.example.com.", type: "A", ttl: 60, data: "192.0.2.1" }, + ]); + }); +}); diff --git a/ui/src/lib/ocidns.ts b/ui/src/lib/ocidns.ts new file mode 100644 index 0000000000..84e116d9ff --- /dev/null +++ b/ui/src/lib/ocidns.ts @@ -0,0 +1,454 @@ +/** Client for the Oracle Cloud DNS emulator through the dashboard's in-process proxy. */ + +import { ociCompartments, ociListPages, readStoredValue, writeStoredValue } from "./ocirest.ts"; +import { GcpRestError, restCall, str, type FetchFn, type Json } from "./gcprest.ts"; + +export const DNS_PROXY_BASE = "/dashboard/api/v1/oci/dns"; +export const DNS_COMPARTMENT_KEY = "gopherstack_ocidns_compartment"; + +export const RECORD_TYPES = ["A", "AAAA", "CNAME", "MX", "NS", "TXT", "SRV", "PTR", "CAA"] as const; +export const QUERY_TYPES = [...RECORD_TYPES, "SOA"] as const; +export const DEFAULT_TTL = 300; + +const enc = encodeURIComponent; +const num = (v: unknown): number => (typeof v === "number" ? v : Number(v) || 0); +const obj = (v: unknown): Json => (v && typeof v === "object" ? (v as Json) : {}); +const list = (v: unknown): Json[] => (Array.isArray(v) ? (v as Json[]) : []); + +export type Scope = "GLOBAL" | "PRIVATE"; + +export type Zone = { + id: string; + name: string; + type: string; + scope: string; + serial: number; + created: string; + state: string; + viewId: string; + nameservers: string[]; +}; + +export type DnsRecord = { + domain: string; + rtype: string; + ttl: number; + rdata: string; + protected: boolean; +}; + +export type RRSet = { domain: string; rtype: string; ttl: number; records: DnsRecord[] }; + +export type View = { id: string; name: string; created: string; protected: boolean; state: string }; +export type Resolver = { + id: string; + name: string; + vcnId: string; + defaultViewId: string; + state: string; + created: string; +}; +export type Endpoint = { + name: string; + subnetId: string; + kind: string; + address: string; + state: string; +}; +export type TsigKey = { + id: string; + name: string; + algorithm: string; + state: string; + created: string; +}; +export type SteeringPolicy = { + id: string; + name: string; + template: string; + ttl: number; + state: string; + created: string; +}; + +export type DnsAnswerRecord = { name: string; type: string; ttl: number; data: string }; +export type DnsAnswer = { + rcode: string; + authoritative: boolean; + name: string; + type: string; + server: string; + answer: DnsAnswerRecord[]; + authority: DnsAnswerRecord[]; +}; + +export type ZoneRecords = { records: DnsRecord[]; etag: string }; + +export type NewZone = { + name: string; + compartmentId: string; + scope: Scope; + viewId: string; +}; + +export type RecordDraft = { domain: string; rtype: string; ttl: number; rdata: string }; +export type RecordOp = { operation: "ADD" | "REMOVE" } & RecordDraft; + +export function parseZone(raw: Json): Zone { + return { + id: str(raw.id), + name: str(raw.name), + type: str(raw.zoneType), + scope: str(raw.scope), + serial: num(raw.serial), + created: str(raw.timeCreated), + state: str(raw.lifecycleState), + viewId: str(raw.viewId), + nameservers: list(raw.nameservers).map((n) => str(n.hostname)), + }; +} + +export function parseRecord(raw: Json): DnsRecord { + return { + domain: str(raw.domain), + rtype: str(raw.rtype), + ttl: num(raw.ttl), + rdata: str(raw.rdata), + protected: raw.isProtected === true, + }; +} + +/** Groups records by domain and type, the way OCI addresses an rrset. */ +export function groupRecords(records: readonly DnsRecord[]): RRSet[] { + const sets = new Map(); + for (const r of records) { + const key = `${r.domain}|${r.rtype}`; + const set = sets.get(key) ?? { domain: r.domain, rtype: r.rtype, ttl: r.ttl, records: [] }; + set.records.push(r); + sets.set(key, set); + } + return [...sets.values()].toSorted( + (a, b) => a.domain.localeCompare(b.domain) || a.rtype.localeCompare(b.rtype), + ); +} + +export const isConflict = (e: unknown): boolean => + e instanceof GcpRestError && (e.httpStatus === 412 || e.httpStatus === 409); + +export function validateRecord(zone: string, d: RecordDraft): string { + const domain = d.domain.trim().replace(/\.$/, ""); + if (domain === "") return "Domain is required"; + if (!/^[A-Za-z0-9_*.-]{1,255}$/.test(domain)) return "Domain has invalid characters"; + if (domain !== zone && !domain.endsWith(`.${zone}`)) + return `Domain must be ${zone} or end in .${zone}`; + if (!(RECORD_TYPES as readonly string[]).includes(d.rtype)) return "Pick a record type"; + if (d.rdata.trim() === "") return "Record data is required"; + if (!Number.isInteger(d.ttl) || d.ttl < 0 || d.ttl > 2147483647) + return "TTL must be a whole number of seconds"; + return ""; +} + +const normalized = (d: RecordDraft): RecordDraft => ({ + domain: d.domain.trim().replace(/\.$/, ""), + rtype: d.rtype, + ttl: d.ttl, + rdata: d.rdata.trim(), +}); + +export const addOp = (d: RecordDraft): RecordOp => ({ operation: "ADD", ...normalized(d) }); +export const removeOp = (d: RecordDraft): RecordOp => ({ operation: "REMOVE", ...normalized(d) }); + +/** An edit is the old record removed and the new one added in one atomic patch. */ +export const editOps = (old: RecordDraft, next: RecordDraft): RecordOp[] => [ + removeOp(old), + addOp(next), +]; + +type Fetched = { body: unknown; etag: string; next: string }; + +async function dnsFetch( + path: string, + init: RequestInit | undefined, + fetchFn: FetchFn, +): Promise { + let res: Response; + try { + res = await fetchFn(`${DNS_PROXY_BASE}${path}`, init); + } catch (e) { + throw new GcpRestError(0, "NetworkError", e instanceof Error ? e.message : String(e)); + } + const text = await res.text(); + let body: unknown = null; + try { + body = text ? JSON.parse(text) : null; + } catch { + // non-JSON error body: keep the status text + } + if (!res.ok) { + const b = obj(body); + const proxy = obj(b.error); + throw new GcpRestError( + res.status, + str(proxy.status) || str(b.code), + str(proxy.message) || str(b.message) || res.statusText || `HTTP ${res.status}`, + ); + } + return { + body, + etag: res.headers.get("etag") ?? "", + next: res.headers.get("opc-next-page") ?? "", + }; +} + +export const listCompartments = (fetchFn: FetchFn = fetch) => + ociCompartments(DNS_PROXY_BASE, fetchFn); + +export async function getDataPlane( + fetchFn: FetchFn = fetch, +): Promise<{ enabled: boolean; port: number }> { + const body = await restCall(DNS_PROXY_BASE, "GET", "/dataplane", undefined, fetchFn); + return { enabled: body.enabled === true, port: num(body.port) }; +} + +const compQuery = (compartment: string) => `compartmentId=${enc(compartment)}`; + +/** Zones of both scopes: GLOBAL zones answer on the data plane, PRIVATE zones belong to a view. */ +export async function listZones(compartment: string, fetchFn: FetchFn = fetch): Promise { + const scopes: Scope[] = ["GLOBAL", "PRIVATE"]; + const lists = await Promise.all( + scopes.map((scope) => + ociListPages( + DNS_PROXY_BASE, + "/api/zones", + `${compQuery(compartment)}&scope=${scope}`, + "", + fetchFn, + ), + ), + ); + return lists + .flat() + .map((z) => parseZone(z)) + .filter((z) => z.state !== "DELETED") + .toSorted((a, b) => a.name.localeCompare(b.name)); +} + +const zoneQuery = (z: Pick) => + z.scope === "PRIVATE" ? `scope=PRIVATE&viewId=${enc(z.viewId)}` : ""; + +export async function getZoneRecords( + zone: Pick, + fetchFn: FetchFn = fetch, +): Promise { + const records: DnsRecord[] = []; + let etag = ""; + let page = ""; + do { + const qs = [zoneQuery(zone), page ? `page=${enc(page)}` : ""].filter(Boolean).join("&"); + const res = await dnsFetch( + `/api/zones/${enc(zone.id)}/records${qs ? `?${qs}` : ""}`, + undefined, + fetchFn, + ); + etag = etag || res.etag; + records.push(...list(obj(res.body).items).map((r) => parseRecord(r))); + page = res.next; + } while (page); + return { records, etag }; +} + +/** Applies ADD/REMOVE operations; the etag makes a concurrent change fail with a conflict. */ +export async function patchRecords( + zone: Pick, + ops: readonly RecordOp[], + etag: string, + fetchFn: FetchFn = fetch, +): Promise { + const qs = zoneQuery(zone); + const headers: Record = { "Content-Type": "application/json" }; + if (etag) headers["If-Match"] = etag; + await dnsFetch( + `/api/zones/${enc(zone.id)}/records${qs ? `?${qs}` : ""}`, + { method: "PATCH", headers, body: JSON.stringify({ items: ops }) }, + fetchFn, + ); +} + +export function createZoneBody(z: NewZone): Json { + const body: Json = { + name: z.name.trim(), + compartmentId: z.compartmentId, + zoneType: "PRIMARY", + scope: z.scope, + }; + if (z.scope === "PRIVATE") body.viewId = z.viewId; + return body; +} + +export function validateZone(z: NewZone): string { + const name = z.name.trim(); + if (name === "") return "Zone name is required"; + if (!/^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)+$/.test(name)) + return "Zone name must be a domain such as example.com"; + if (z.scope === "PRIVATE" && z.viewId === "") return "A private zone needs a view"; + return ""; +} + +export async function createZone(z: NewZone, fetchFn: FetchFn = fetch): Promise { + await restCall(DNS_PROXY_BASE, "POST", "/api/zones", createZoneBody(z), fetchFn); +} + +export async function deleteZone( + zone: Pick, + fetchFn: FetchFn = fetch, +): Promise { + const qs = zoneQuery(zone); + await restCall( + DNS_PROXY_BASE, + "DELETE", + `/api/zones/${enc(zone.id)}${qs ? `?${qs}` : ""}`, + undefined, + fetchFn, + ); +} + +export async function listViews(compartment: string, fetchFn: FetchFn = fetch): Promise { + const items = await ociListPages( + DNS_PROXY_BASE, + "/api/views", + compQuery(compartment), + "", + fetchFn, + ); + return items.map((v) => ({ + id: str(v.id), + name: str(v.displayName), + created: str(v.timeCreated), + protected: v.isProtected === true, + state: str(v.lifecycleState), + })); +} + +export async function listResolvers( + compartment: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DNS_PROXY_BASE, + "/api/resolvers", + compQuery(compartment), + "", + fetchFn, + ); + return items.map((r) => ({ + id: str(r.id), + name: str(r.displayName), + vcnId: str(r.attachedVcnId), + defaultViewId: str(r.defaultViewId), + state: str(r.lifecycleState), + created: str(r.timeCreated), + })); +} + +export async function listEndpoints( + resolverId: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DNS_PROXY_BASE, + `/api/resolvers/${enc(resolverId)}/endpoints`, + "", + "", + fetchFn, + ); + return items.map((e) => ({ + name: str(e.name), + subnetId: str(e.subnetId), + kind: e.isForwarding === true ? "Forwarding" : "Listening", + address: str(e.forwardingAddress) || str(e.listeningAddress), + state: str(e.lifecycleState), + })); +} + +/** TSIG keys as the proxy answers them: the secret is never part of the answer. */ +export async function listTsigKeys( + compartment: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DNS_PROXY_BASE, + "/api/tsigKeys", + compQuery(compartment), + "", + fetchFn, + ); + return items.map((k) => ({ + id: str(k.id), + name: str(k.name), + algorithm: str(k.algorithm), + state: str(k.lifecycleState), + created: str(k.timeCreated), + })); +} + +export async function listSteeringPolicies( + compartment: string, + fetchFn: FetchFn = fetch, +): Promise { + const items = await ociListPages( + DNS_PROXY_BASE, + "/api/steeringPolicies", + compQuery(compartment), + "", + fetchFn, + ); + return items.map((p) => ({ + id: str(p.id), + name: str(p.displayName), + template: str(p.template), + ttl: num(p.ttl), + state: str(p.lifecycleState), + created: str(p.timeCreated), + })); +} + +const parseRR = (r: Json): DnsAnswerRecord => ({ + name: str(r.name), + type: str(r.type), + ttl: num(r.ttl), + data: str(r.data), +}); + +export function validateQuery(name: string, type: string): string { + const n = name.trim().replace(/\.$/, ""); + if (n === "") return "Enter a name to look up"; + if ( + n.length > 253 || + !n.split(".").every((l) => /^[A-Za-z0-9_*]([A-Za-z0-9_-]{0,61}[A-Za-z0-9_])?$/.test(l)) + ) { + return "Not a valid DNS name"; + } + if (!(QUERY_TYPES as readonly string[]).includes(type)) return "Pick a record type"; + return ""; +} + +/** Runs one query against the emulator's DNS data plane (server side, loopback only). */ +export async function tryQuery( + name: string, + type: string, + fetchFn: FetchFn = fetch, +): Promise { + const b = await restCall(DNS_PROXY_BASE, "POST", "/query", { name: name.trim(), type }, fetchFn); + return { + rcode: str(b.rcode), + authoritative: b.authoritative === true, + name: str(b.name), + type: str(b.type), + server: str(b.server), + answer: list(b.answer).map((r) => parseRR(r)), + authority: list(b.authority).map((r) => parseRR(r)), + }; +} + +export const readStoredDnsCompartment = () => readStoredValue(DNS_COMPARTMENT_KEY); +export const writeStoredDnsCompartment = (id: string) => writeStoredValue(DNS_COMPARTMENT_KEY, id); diff --git a/ui/src/routes/azure/[service]/+page.svelte b/ui/src/routes/azure/[service]/+page.svelte index 377153d37d..b63e55b1a0 100644 --- a/ui/src/routes/azure/[service]/+page.svelte +++ b/ui/src/routes/azure/[service]/+page.svelte @@ -5,6 +5,7 @@ import ApiManagementDashboard from '#lib/components/azure/ApiManagementDashboard.svelte'; import AppServiceDashboard from '#lib/components/azure/AppServiceDashboard.svelte'; import ArmDashboard from '#lib/components/azure/ArmDashboard.svelte'; + import EntraDashboard from '#lib/components/azure/EntraDashboard.svelte'; import FlexibleServerDashboard from '#lib/components/azure/FlexibleServerDashboard.svelte'; import KubernetesServiceDashboard from '#lib/components/azure/KubernetesServiceDashboard.svelte'; import ContainerAppsDashboard from '#lib/components/azure/ContainerAppsDashboard.svelte'; @@ -21,6 +22,7 @@ import QueueDashboard from '#lib/components/azure/QueueDashboard.svelte'; import ServiceBusDashboard from '#lib/components/azure/ServiceBusDashboard.svelte'; import TableDashboard from '#lib/components/azure/TableDashboard.svelte'; + import NetworkingDashboard from '#lib/components/azure/NetworkingDashboard.svelte'; import VirtualMachinesDashboard from '#lib/components/azure/VirtualMachinesDashboard.svelte'; import { backendInfo, loadBackendInfo, servedAzureServices } from '#lib/cloud.svelte.ts'; @@ -74,6 +76,10 @@ {:else if service?.id === 'apim'} +{:else if service?.id === 'network'} + +{:else if service?.id === 'entra'} + {:else}
{#if !service} diff --git a/ui/src/routes/oci/[service]/+page.svelte b/ui/src/routes/oci/[service]/+page.svelte index 8655a75611..42f5833716 100644 --- a/ui/src/routes/oci/[service]/+page.svelte +++ b/ui/src/routes/oci/[service]/+page.svelte @@ -7,6 +7,8 @@ import ContainerInstancesDashboard from '#lib/components/oci/ContainerInstancesDashboard.svelte'; import ContainerEngineDashboard from '#lib/components/oci/ContainerEngineDashboard.svelte'; import CoreDashboard from '#lib/components/oci/CoreDashboard.svelte'; + import DatabaseDashboard from '#lib/components/oci/DatabaseDashboard.svelte'; + import DnsDashboard from '#lib/components/oci/DnsDashboard.svelte'; import EventsDashboard from '#lib/components/oci/EventsDashboard.svelte'; import FunctionsDashboard from '#lib/components/oci/FunctionsDashboard.svelte'; import IdentityDashboard from '#lib/components/oci/IdentityDashboard.svelte'; @@ -68,6 +70,10 @@ {:else if service?.id === 'containerinstances'} +{:else if service?.id === 'dns'} + +{:else if service?.id === 'database'} + {:else}
{#if !service}