diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2590e6..f68980c 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,48 @@ jobs: with: version: 3.21.3 + - name: Lint and render Helm chart + run: | + set -euo pipefail + helm lint deploy/rustfs-operator + + # Default: cosiDriver disabled, nothing COSI-related should render. + helm template test deploy/rustfs-operator > /tmp/helm-default.yaml + ! grep -q "cosi-driver" /tmp/helm-default.yaml + + # cosiDriver enabled on vanilla Kubernetes. + helm template test deploy/rustfs-operator \ + --set cosiDriver.enabled=true > /tmp/helm-cosi.yaml + grep -q "kind: Deployment" /tmp/helm-cosi.yaml + grep -q "rustfs-cosi-driver" /tmp/helm-cosi.yaml + grep -q "objectstorage-sidecar" /tmp/helm-cosi.yaml + + # cosiDriver enabled on OpenShift: no hardcoded securityContext, hostUsers set. + helm template test deploy/rustfs-operator \ + --set cosiDriver.enabled=true --set openshift.enabled=true \ + --show-only templates/cosi-driver-deployment.yaml > /tmp/helm-cosi-openshift.yaml + grep -q "hostUsers: false" /tmp/helm-cosi-openshift.yaml + ! grep -q "runAsUser" /tmp/helm-cosi-openshift.yaml + + # Custom ServiceAccount: chart must not create its own (a disabled template + # renders nothing, so assert on the full render rather than --show-only, + # which errors on a template that produced no manifest), and the Deployment + # must reference the supplied name. + helm template test deploy/rustfs-operator \ + --set cosiDriver.enabled=true \ + --set cosiDriver.serviceAccount.create=false \ + --set cosiDriver.serviceAccount.name=my-custom-sa \ + > /tmp/helm-cosi-custom-sa.yaml + ! grep -q "cosi-driver-serviceaccount.yaml" /tmp/helm-cosi-custom-sa.yaml + grep -q "serviceAccountName: my-custom-sa" /tmp/helm-cosi-custom-sa.yaml + + # Custom RBAC: chart must not create its own ClusterRole/ClusterRoleBinding. + helm template test deploy/rustfs-operator \ + --set cosiDriver.enabled=true \ + --set cosiDriver.rbac.create=false \ + > /tmp/helm-cosi-custom-rbac.yaml + ! grep -q "cosi-driver-clusterrole" /tmp/helm-cosi-custom-rbac.yaml + - name: Check release metadata run: make release-metadata-check diff --git a/CHANGELOG.md b/CHANGELOG.md index 41daafe..0e8efe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ All notable changes to RustFS Operator are documented in this file. The format i for operator observability, STS, and Console sockets. - Tenant `spec.hostUsers` and OpenShift `hostUsers: false` defaults for `restricted-v3`. - Tenant bucket canned anonymous access and ConfigMap-sourced bucket policies. +- Helm chart `cosiDriver.enabled` to deploy the RustFS COSI (`rustfs.objectstorage.k8s.io`) + driver alongside the upstream provisioner sidecar, with dedicated ServiceAccount/RBAC. ### Fixed diff --git a/deploy/rustfs-operator/templates/_helpers.tpl b/deploy/rustfs-operator/templates/_helpers.tpl index 01c98d6..c7dfdb8 100755 --- a/deploy/rustfs-operator/templates/_helpers.tpl +++ b/deploy/rustfs-operator/templates/_helpers.tpl @@ -102,3 +102,14 @@ Create the name of the console service account to use {{- default "default" .Values.console.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Create the name of the COSI driver service account to use +*/}} +{{- define "rustfs-operator.cosiDriverServiceAccountName" -}} +{{- if .Values.cosiDriver.serviceAccount.create }} +{{- default (printf "%s-cosi-driver" (include "rustfs-operator.fullname" .)) .Values.cosiDriver.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.cosiDriver.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-driver-clusterrole.yaml b/deploy/rustfs-operator/templates/cosi-driver-clusterrole.yaml new file mode 100644 index 0000000..7821957 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-driver-clusterrole.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.cosiDriver.enabled .Values.cosiDriver.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi-driver + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi-driver +rules: + # Upstream COSI sidecar: reconciles BucketClaim/BucketAccess against BucketClass/ + # BucketAccessClass by calling this driver's Identity/Provisioner gRPC service. + - apiGroups: ["objectstorage.k8s.io"] + resources: + - "buckets" + - "bucketaccesses" + - "bucketclaims" + - "bucketaccessclasses" + - "buckets/status" + - "bucketaccesses/status" + - "bucketclaims/status" + - "bucketaccessclasses/status" + verbs: ["get", "list", "watch", "update", "create", "delete"] + + # Upstream COSI sidecar: leader election. + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + + # Upstream COSI sidecar and RustFS driver: BucketAccess credential Secrets and the + # RustFS admin Secret. No `list`/`watch`: both only ever address Secrets by name. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "delete"] + + # Upstream COSI sidecar: event recording. client-go's event recorder only ever + # creates new Events and PATCHes repeated/aggregated ones (no list/watch/delete). + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + + # RustFS driver: reads the RustFS admin Secret and optional TLS CA ConfigMap named by + # BucketClass/BucketAccessClass parameters, and stores compare-and-swap grant/bucket + # ownership records as ConfigMaps. + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "delete"] +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-driver-clusterrolebinding.yaml b/deploy/rustfs-operator/templates/cosi-driver-clusterrolebinding.yaml new file mode 100644 index 0000000..4ba3d50 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-driver-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.cosiDriver.enabled .Values.cosiDriver.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi-driver + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "rustfs-operator.fullname" . }}-cosi-driver +subjects: + - kind: ServiceAccount + name: {{ include "rustfs-operator.cosiDriverServiceAccountName" . }} + namespace: {{ include "rustfs-operator.namespace" . }} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-driver-deployment.yaml b/deploy/rustfs-operator/templates/cosi-driver-deployment.yaml new file mode 100644 index 0000000..1048b2a --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-driver-deployment.yaml @@ -0,0 +1,108 @@ +{{- if .Values.cosiDriver.enabled -}} +{{- $openShift := default dict .Values.openshift -}} +{{- $openShiftEnabled := default false $openShift.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "rustfs-operator.fullname" . }}-cosi-driver + namespace: {{ include "rustfs-operator.namespace" . }} + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi-driver + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.cosiDriver.replicas }} + selector: + matchLabels: + {{- include "rustfs-operator.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: cosi-driver + template: + metadata: + labels: + {{- include "rustfs-operator.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: cosi-driver + spec: + serviceAccountName: {{ include "rustfs-operator.cosiDriverServiceAccountName" . }} + {{- if $openShiftEnabled }} + hostUsers: false + {{- end }} + {{- with .Values.cosiDriver.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not $openShiftEnabled }} + {{- with .Values.cosiDriver.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + containers: + # RustFS driver: implements the COSI Identity + Provisioner gRPC service on a + # Unix socket shared with the sidecar container below. + - name: rustfs-driver + image: "{{ .Values.cosiDriver.image.repository }}:{{ .Values.cosiDriver.image.tag | default .Values.operator.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.cosiDriver.image.pullPolicy }} + command: ["./rustfs-cosi-driver"] + env: + - name: RUST_LOG + value: {{ .Values.cosiDriver.logLevel | default "info" }} + - name: COSI_ENDPOINT + value: "unix:///var/lib/cosi/cosi.sock" + volumeMounts: + - name: socket + mountPath: /var/lib/cosi + {{- with .Values.cosiDriver.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if not $openShiftEnabled }} + {{- with .Values.cosiDriver.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + + # Upstream COSI provisioner sidecar: watches BucketClaim/BucketAccess and calls + # the driver above over the shared Unix socket. + - name: objectstorage-provisioner-sidecar + image: "{{ .Values.cosiDriver.sidecar.image.repository }}:{{ .Values.cosiDriver.sidecar.image.tag }}" + imagePullPolicy: {{ .Values.cosiDriver.sidecar.image.pullPolicy }} + args: + - "--v={{ .Values.cosiDriver.sidecar.verbosity }}" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: socket + mountPath: /var/lib/cosi + {{- with .Values.cosiDriver.sidecar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if not $openShiftEnabled }} + {{- with .Values.cosiDriver.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + volumes: + - name: socket + emptyDir: {} + {{- with .Values.cosiDriver.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosiDriver.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosiDriver.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/rustfs-operator/templates/cosi-driver-serviceaccount.yaml b/deploy/rustfs-operator/templates/cosi-driver-serviceaccount.yaml new file mode 100644 index 0000000..3a83c04 --- /dev/null +++ b/deploy/rustfs-operator/templates/cosi-driver-serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.cosiDriver.enabled .Values.cosiDriver.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "rustfs-operator.cosiDriverServiceAccountName" . }} + namespace: {{ include "rustfs-operator.namespace" . }} + labels: + {{- include "rustfs-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: cosi-driver + {{- with .Values.cosiDriver.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/rustfs-operator/values.yaml b/deploy/rustfs-operator/values.yaml index 1c88bbd..b2a157d 100755 --- a/deploy/rustfs-operator/values.yaml +++ b/deploy/rustfs-operator/values.yaml @@ -333,3 +333,97 @@ console: # - secretName: console-tls # hosts: # - console.example.com + +# RustFS COSI (Container Object Storage Interface) v1alpha1 driver. +# +# Requires the upstream COSI CRDs and controller +# (https://github.com/kubernetes-sigs/container-object-storage-interface) to already be +# installed in the cluster; this chart only deploys the RustFS driver + provisioner sidecar +# Deployment that implements the COSI Identity/Provisioner gRPC service. +# +# Driver name for BucketClass / BucketAccessClass: rustfs.objectstorage.k8s.io +cosiDriver: + # Disabled by default because it depends on cluster-wide COSI CRDs/controller as a + # prerequisite; enable once those are installed. + enabled: false + + replicas: 1 + + # Log level for the driver (trace, debug, info, warn, error) + logLevel: info + + image: + # The COSI driver binary ships in the same image as operator. + repository: rustfs/operator + tag: "" # Defaults to operator.image.tag + pullPolicy: IfNotPresent + + # Image pull secrets for private registries + imagePullSecrets: [] + + # Official COSI provisioner sidecar. It watches BucketClaim/BucketAccess/BucketClass/ + # BucketAccessClass objects and calls the RustFS driver's Identity/Provisioner gRPC + # service over a Unix socket shared via an emptyDir volume. + sidecar: + image: + repository: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar + tag: v0.2.2 + pullPolicy: IfNotPresent + # klog verbosity (0-5) + verbosity: 4 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + + # Resource limits and requests for the driver container + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + + # Security context for the driver pod + podSecurityContext: + fsGroup: 65534 + + # Security context for the driver container (also applied to the sidecar container) + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + + # Node selector for driver pod placement + nodeSelector: {} + + # Tolerations for driver pod scheduling + tolerations: [] + + # Affinity rules for driver pod scheduling + affinity: {} + + # ServiceAccount configuration + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + name: "" + + # RBAC configuration. The ClusterRole covers both the RustFS driver (Secrets/ConfigMaps + # for credential and CAS-ownership records) and the upstream sidecar (objectstorage.k8s.io + # CRDs and coordination.k8s.io Leases for leader election). + rbac: + # Specifies whether RBAC resources should be created + create: true diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index 0dc9c1a..202f154 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -1239,6 +1239,108 @@ When using the RustFS COSI driver (`rustfs.objectstorage.k8s.io`): - If you set `preferredAccessKey` (or `accessKey`), the value must be unique per `BucketAccess`. Reusing the same key across claims is rejected with `AlreadyExists` so credentials are never rotated out from under another workload. - Grant retries for the same `BucketAccess` are idempotent and return the same secret; the driver does not overwrite an existing user's secret key. +## 13.2 Deploying the COSI driver + +The RustFS COSI driver ships in the same operator image (`./rustfs-cosi-driver`) but is not +deployed by default. It requires the upstream +[COSI CRDs and controller](https://github.com/kubernetes-sigs/container-object-storage-interface) +(API version `objectstorage.k8s.io/v1alpha1`) to already be installed in the cluster; the chart's +`cosiDriver.sidecar.image.tag` defaults to the matching sidecar release, `v0.2.2`. + +Both the RustFS driver Deployment and the upstream controller/sidecar images are plain +non-privileged containers with no `hostNetwork`, `hostPath`, or fixed-UID requirement in their +Kubernetes manifests (the upstream images declare a non-root user only in their Dockerfile, not +the pod spec, so an platform-assigned UID applies cleanly). The only platform-specific step is +enabling this chart's existing `openshift.enabled` flag, which already governs every other +Deployment in this chart the same way. This is verified by manifest/Dockerfile inspection of the +pinned upstream versions, not a live OpenShift smoke test — validate in a scratch project before +relying on it in production. + +### Vanilla Kubernetes + +1. Install the upstream COSI CRDs and controller (once per cluster): + + ```bash + kubectl apply -k "github.com/kubernetes-sigs/container-object-storage-interface?ref=v0.2.2" + ``` + + This also creates the controller's own `ServiceAccount` in the `default` namespace (an + upstream kustomize limitation, not specific to this chart). + +2. Enable the driver Deployment in the Helm chart: + + ```yaml + cosiDriver: + enabled: true + ``` + +### OpenShift + +1. Install the upstream COSI CRDs and controller, exactly as above — no SCC binding or + additional `oc adm policy` steps are needed; the controller and sidecar run fine under the + default `restricted-v2` SCC: + + ```bash + oc apply -k "github.com/kubernetes-sigs/container-object-storage-interface?ref=v0.2.2" + ``` + +2. Enable the driver Deployment alongside this chart's existing OpenShift toggle, so the + RustFS driver container and the sidecar container it shares a pod with also drop their + explicit `securityContext`/`runAsUser` and let the SCC assign the pod's UID: + + ```yaml + openshift: + enabled: true + cosiDriver: + enabled: true + ``` + +Either way, this deploys a Deployment with two containers sharing a Unix socket: the RustFS +driver (Identity + Provisioner gRPC) and the upstream `objectstorage-sidecar` container that +watches `BucketClaim`/`BucketAccess` objects. A dedicated `ServiceAccount` and `ClusterRole` are +created covering both containers' Kubernetes API access (`objectstorage.k8s.io` resources for the +sidecar; `Secrets`/`ConfigMaps` for the driver's credential and ownership records). Set +`cosiDriver.rbac.create: false` or `cosiDriver.serviceAccount.create: false` to supply your own. + +### Provisioning a bucket + +3. Create a `BucketClass` and `BucketAccessClass` with `driverName: rustfs.objectstorage.k8s.io` + and `authenticationType: Key` (the only mode this driver supports), pointing `endpoint` / + `objectStoreUserSecretName` / `objectStoreUserSecretNamespace` at a Tenant's S3 endpoint and + an admin credentials Secret. Then create a `BucketClaim` and a `BucketAccess` referencing it, + with an explicit `spec.protocol: S3` — `objectstorage-sidecar` v0.2.2 copies this field + verbatim into the generated credentials and does not implement the CRD's documented + fallback to the bucket's supported protocols, so an omitted value produces an empty + protocol. See + [examples/cosi-bucket-provisioning.yaml](../examples/cosi-bucket-provisioning.yaml) for a + full walkthrough, including a workload Pod consuming the resulting Secret. +4. Once `BucketAccess.status.accessGranted` is `true`, the Secret named by + `credentialsSecretName` exists in the `BucketAccess`'s namespace — **not** as flat keys, but + as a single key, `BucketInfo`, whose value is a JSON document (this is + `objectstorage-sidecar` v0.2.2's own Secret format, not something the RustFS driver + controls): + + ```json + { + "spec": { + "bucketName": "", + "authenticationType": "Key", + "secretS3": { + "endpoint": "", + "region": "", + "accessKeyID": "", + "accessSecretKey": "" + }, + "protocols": ["S3"] + } + } + ``` + + A workload must mount this Secret as a file and parse the JSON — `envFrom.secretRef` does + not produce usable flat environment variables, since there is only one key. See + [examples/cosi-bucket-provisioning.yaml](../examples/cosi-bucket-provisioning.yaml) for a + Pod mounting and reading it. + ## 14. Related Documentation - [Project README](../README.md) diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index 7c11161..7d8da8b 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -1186,6 +1186,106 @@ RustFS Tenant Console 登录失败时，应使用 `spec.credsSecret` 或 RustFS - Tenant YAML 可以进入版本控制，但不要提交明文 Secret 值。 - 优先查看 `status.conditions`，再进一步排查 StatefulSet 和 Pod。 +## 13.1 COSI `preferredAccessKey` + +使用 RustFS COSI 驱动（`rustfs.objectstorage.k8s.io`）时： + +- 建议省略 `preferredAccessKey`，让每个 `BucketAccess` 从 COSI 授权名称（`ba-`）派生出唯一的账户 ID，与 Ceph COSI 的隔离方式保持一致。 +- 如果设置了 `preferredAccessKey`（或 `accessKey`），该值必须在所有 `BucketAccess` 中唯一；不同申请复用同一个值会被拒绝并返回 `AlreadyExists`，避免凭据在其他工作负载不知情的情况下被轮换。 +- 同一个 `BucketAccess` 的重复授权请求是幂等的，会返回相同的 Secret；驱动不会覆盖已有用户的密钥。 + +## 13.2 部署 COSI 驱动 + +RustFS COSI 驱动与 operator 共用同一个镜像（`./rustfs-cosi-driver`），但默认不会部署。它依赖集群中已安装的上游 +[COSI CRD 与控制器](https://github.com/kubernetes-sigs/container-object-storage-interface) +（API 版本 `objectstorage.k8s.io/v1alpha1`）；chart 的 `cosiDriver.sidecar.image.tag` 默认对应 +匹配的 sidecar 发行版本 `v0.2.2`。 + +RustFS 驱动 Deployment 与上游 controller/sidecar 镜像都是普通的非特权容器，其 Kubernetes 清单中 +没有 `hostNetwork`、`hostPath`，也没有固定 UID 要求（上游镜像仅在 Dockerfile 中声明了非 root 用户， +并未写入 Pod spec，因此可以直接采用平台分配的 UID）。唯一与平台相关的步骤是启用本 chart 已有的 +`openshift.enabled` 开关——它对本 chart 中所有 Deployment 的处理方式是一致的。以下结论基于对所依赖的 +上游版本清单/Dockerfile 的静态分析，并未在真实 OpenShift 集群上做过冒烟测试，正式使用前请先在测试 +项目中验证。 + +### 原生 Kubernetes + +1. 在集群中安装上游 COSI CRD 与控制器（每个集群安装一次即可）： + + ```bash + kubectl apply -k "github.com/kubernetes-sigs/container-object-storage-interface?ref=v0.2.2" + ``` + + 这也会在 `default` 命名空间下创建控制器自己的 `ServiceAccount`（这是上游 kustomize 清单本身的限制， + 与本 chart 无关）。 + +2. 在 Helm chart 中启用驱动 Deployment： + + ```yaml + cosiDriver: + enabled: true + ``` + +### OpenShift + +1. 安装上游 COSI CRD 与控制器，步骤与上面完全相同——不需要额外绑定 SCC 或执行 + `oc adm policy` 命令；controller 和 sidecar 在默认的 `restricted-v2` SCC 下即可正常运行： + + ```bash + oc apply -k "github.com/kubernetes-sigs/container-object-storage-interface?ref=v0.2.2" + ``` + +2. 与本 chart 已有的 OpenShift 开关一起启用驱动 Deployment，这样 RustFS 驱动容器及与其共享 Pod 的 + sidecar 容器都会省略显式的 `securityContext`/`runAsUser`，交由 SCC 分配 Pod 的 UID： + + ```yaml + openshift: + enabled: true + cosiDriver: + enabled: true + ``` + +无论哪种平台，这都会部署一个包含两个容器、共享 Unix socket 的 Deployment：RustFS 驱动 +（Identity + Provisioner gRPC）和上游 `objectstorage-sidecar` 容器（负责监听 +`BucketClaim`/`BucketAccess`）。同时会创建专用的 `ServiceAccount` 和 `ClusterRole`，覆盖两个容器所需的 +Kubernetes API 权限（sidecar 需要 `objectstorage.k8s.io` 相关资源；驱动需要 `Secrets`/`ConfigMaps` +以存取凭据和所有权记录）。如需自行提供，可将 `cosiDriver.rbac.create` 或 +`cosiDriver.serviceAccount.create` 设为 `false`。 + +### 创建一个桶 + +3. 创建 `driverName` 为 `rustfs.objectstorage.k8s.io`、`authenticationType` 为 `Key`（驱动仅支持这一种 + 认证方式）的 `BucketClass` 和 `BucketAccessClass`，将 `endpoint`、`objectStoreUserSecretName`、 + `objectStoreUserSecretNamespace` 指向某个 Tenant 的 S3 端点及其管理员凭据 Secret，再创建引用它们的 + `BucketClaim` 和 `BucketAccess`，并显式设置 `spec.protocol: S3`——`objectstorage-sidecar` + v0.2.2 会原样将该字段写入生成的凭据，并未实现 CRD 文档中所述“回退到桶支持的协议”的逻辑，省略该字段会 + 导致协议值为空。完整示例（含消费生成 Secret 的工作负载 Pod）参见 + [examples/cosi-bucket-provisioning.yaml](../examples/cosi-bucket-provisioning.yaml)。 +4. 当 `BucketAccess.status.accessGranted` 变为 `true` 后，`credentialsSecretName` 指定的 Secret + 会在该 `BucketAccess` 所在命名空间下生成——**并不是**扁平的键值对，而是单独一个键 + `BucketInfo`，其值是一段 JSON（这是 `objectstorage-sidecar` v0.2.2 自身的 Secret 格式，并非 + RustFS 驱动决定的）： + + ```json + { + "spec": { + "bucketName": "<生成的 S3 桶名称>", + "authenticationType": "Key", + "secretS3": { + "endpoint": "", + "region": "", + "accessKeyID": "", + "accessSecretKey": "" + }, + "protocols": ["S3"] + } + } + ``` + + 工作负载必须将该 Secret 挂载为文件并解析这段 JSON——由于只有一个键，`envFrom.secretRef` + 无法得到可直接使用的扁平环境变量。挂载并读取该文件的 Pod 示例参见 + [examples/cosi-bucket-provisioning.yaml](../examples/cosi-bucket-provisioning.yaml)。 + ## 14. 相关文档 - [项目 README](../README.md) diff --git a/examples/README.md b/examples/README.md index eddcd7a..f2bdd13 100755 --- a/examples/README.md +++ b/examples/README.md @@ -27,6 +27,12 @@ This directory contains example Tenant configurations for the RustFS Kubernetes 2. Read **simple-tenant.yaml** to understand all options 3. Explore other examples based on your use case +## COSI (Container Object Storage Interface) + +| Example | Use Case | +|---------|----------| +| [cosi-bucket-provisioning.yaml](./cosi-bucket-provisioning.yaml) | `BucketClass`/`BucketAccessClass`/`BucketClaim`/`BucketAccess` against a Tenant, via the `rustfs.objectstorage.k8s.io` driver (enable with `cosiDriver.enabled: true` in the Helm chart) | + On OpenShift, use **openshift-tenant.yaml** only with an arbitrary-UID-compatible RustFS image. Its two explicit empty Pool security contexts form one delegation signal for UID, GID, FSGroup, and container security settings. Both objects are diff --git a/examples/cosi-bucket-provisioning.yaml b/examples/cosi-bucket-provisioning.yaml new file mode 100644 index 0000000..a95cc8c --- /dev/null +++ b/examples/cosi-bucket-provisioning.yaml @@ -0,0 +1,120 @@ +# Example: provisioning an S3 bucket and scoped credentials against a RustFS +# Tenant through the Kubernetes COSI (Container Object Storage Interface) API. +# +# Prerequisites (not created by this file or by the rustfs-operator chart): +# 1. The upstream COSI CRDs and controller are installed in the cluster: +# https://github.com/kubernetes-sigs/container-object-storage-interface +# 2. The RustFS operator chart is installed with `cosiDriver.enabled: true` +# (see deploy/rustfs-operator/values.yaml), which deploys the +# `rustfs.objectstorage.k8s.io` driver + provisioner sidecar. +# 3. A RustFS Tenant is running, and a Kubernetes Secret in its namespace +# holds RustFS admin credentials this driver will authenticate with +# (e.g. the Tenant's own `spec.credsSecret`, or a dedicated admin user's +# secret with `accesskey`/`secretkey` keys). +# +# Apply order: BucketClass and BucketAccessClass first, then BucketClaim, then +# BucketAccess (BucketAccess references the BucketClaim by name). +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketClass +metadata: + name: rustfs-bucket-class +driverName: rustfs.objectstorage.k8s.io +deletionPolicy: Delete +parameters: + # RustFS S3 endpoint for the target Tenant (in-cluster Service DNS name). + endpoint: "http://my-tenant-hl.rustfs-tenant.svc.cluster.local:9000" + # Secret holding RustFS admin credentials (accesskey/secretkey keys), used by + # the driver to create buckets and manage per-BucketAccess users. + objectStoreUserSecretName: my-tenant-admin-creds + objectStoreUserSecretNamespace: rustfs-tenant + region: us-east-1 + # Optional: TLS CA for a TLS-enabled Tenant endpoint. + # tlsCAConfigMapName: my-tenant-ca-bundle + # tlsCAConfigMapNamespace: rustfs-tenant +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccessClass +metadata: + name: rustfs-bucket-access-class +driverName: rustfs.objectstorage.k8s.io +# The RustFS driver only supports Key (static access/secret key) authentication; +# it rejects IAM-style BucketAccessClasses. +authenticationType: Key +parameters: + endpoint: "http://my-tenant-hl.rustfs-tenant.svc.cluster.local:9000" + objectStoreUserSecretName: my-tenant-admin-creds + objectStoreUserSecretNamespace: rustfs-tenant + region: us-east-1 + # Optional: reference an existing RustFS canned policy instead of a + # driver-generated per-grant policy scoped to this bucket. + # policy: my-existing-policy-name +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketClaim +metadata: + name: my-app-bucket + namespace: my-app +spec: + bucketClassName: rustfs-bucket-class + protocols: + - S3 +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccess +metadata: + name: my-app-bucket-access + namespace: my-app +spec: + bucketClaimName: my-app-bucket + bucketAccessClassName: rustfs-bucket-access-class + # Required in practice: objectstorage-sidecar v0.2.2 copies this value verbatim into + # the generated BucketInfo (see below) and does not fall back to the bucket's + # supported protocols despite the BucketAccess CRD describing the field as optional. + protocol: S3 + # Secret the sidecar will create with a single `BucketInfo` JSON key (see below). + credentialsSecretName: my-app-bucket-credentials +--- +# IMPORTANT: with objectstorage-sidecar v0.2.2, the credentialsSecretName Secret above is +# NOT a set of flat keys like AWS_ACCESS_KEY_ID. Once BucketAccess reports +# status.accessGranted: true, the sidecar writes exactly one key, `BucketInfo`, whose +# value is a JSON document shaped like: +# +# { +# "metadata": {"name": "bucketinfo-"}, +# "spec": { +# "bucketName": "", +# "authenticationType": "Key", +# "secretS3": { +# "endpoint": "", +# "region": "", +# "accessKeyID": "", +# "accessSecretKey": "" +# }, +# "protocols": ["S3"] +# } +# } +# +# A workload must mount and parse this JSON — envFrom/flat env vars do not apply. Mounting +# it as a file is the simplest option: +apiVersion: v1 +kind: Pod +metadata: + name: my-app + namespace: my-app +spec: + containers: + - name: my-app + image: my-app:latest + # Reads $BUCKET_INFO_PATH, parses the JSON, and extracts .spec.secretS3.*. + volumeMounts: + - name: bucket-credentials + mountPath: /var/run/secrets/cosi + readOnly: true + env: + - name: BUCKET_INFO_PATH + value: /var/run/secrets/cosi/BucketInfo + volumes: + - name: bucket-credentials + secret: + secretName: my-app-bucket-credentials