Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 31 additions & 26 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ ifeq ($(origin INSTALLER), undefined)
else
$(MAKE) datamate-$(INSTALLER)-install
$(MAKE) milvus-$(INSTALLER)-install
@rm -f /tmp/datamate-helm-args.sh
endif

.PHONY: uninstall-%
Expand Down Expand Up @@ -357,8 +358,8 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
exit 1; \
fi
@if [ "$*" = "label-studio" ]; then \
kubectl apply -f deployment/kubernetes/sealed-secrets/label-studio.yaml; \
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \
if [ -f /tmp/datamate-helm-args.sh ]; then source /tmp/datamate-helm-args.sh; fi; \
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install $${HELM_LABEL_STUDIO_TOLERATIONS:-}; \
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
elif [ "$*" = "mineru-310P" ]; then \
Expand All @@ -370,32 +371,41 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
if [ -f /tmp/datamate-helm-args.sh ]; then \
source /tmp/datamate-helm-args.sh; \
fi; \
kubectl apply -f deployment/kubernetes/sealed-secrets/datamate.yaml; \
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$$HELM_TOLERATIONS_ARGS" ]; then \
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) --set public.secrets.create=false $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
chmod +x scripts/k8s/collect-secrets.sh; \
eval $$(NAMESPACE=$(NAMESPACE) bash scripts/k8s/collect-secrets.sh); \
if [ "$$SECRETS_CREATE" = "SKIP" ]; then \
echo "[SKIP] Secrets collection failed — skipping datamate Helm install"; \
rm -f /tmp/datamate-helm-args.sh; \
exit 0; \
fi; \
if [ -n "$$HELM_VALUES_FILE" ] && [ -f "$$HELM_VALUES_FILE" ]; then \
HELM_EXTRA_ARGS="-f $$HELM_VALUES_FILE"; \
else \
HELM_EXTRA_ARGS=""; \
fi; \
if [ -n "$$HELM_NODE_SELECTOR_ARGS" ] || [ -n "$$HELM_TOLERATIONS_ARGS" ]; then \
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS $$HELM_NODE_SELECTOR_ARGS $$HELM_TOLERATIONS_ARGS; \
else \
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) --set public.secrets.create=false; \
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --force --set global.image.repository=$(REGISTRY) --set public.secrets.create=$$SECRETS_CREATE --set public.persistentVolumeClaim.accessModes=ReadWriteOnce $$HELM_EXTRA_ARGS; \
fi; \
rm -f /tmp/datamate-helm-args.sh; \
rm -f /tmp/datamate-secret-values-*.yaml; \
elif [ "$*" = "deer-flow" ]; then \
cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \
cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \
helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
elif [ "$*" = "milvus" ]; then \
kubectl apply -f deployment/kubernetes/sealed-secrets/milvus.yaml 2>/dev/null || true; \
ACCESSKEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accessKey}' 2>/dev/null | base64 -d 2>/dev/null || echo ""); \
SECRETKEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretKey}' 2>/dev/null | base64 -d 2>/dev/null || echo ""); \
if [ -n "$$ACCESSKEY" ] && [ -n "$$SECRETKEY" ]; then \
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
--set minio.accessKey=$$ACCESSKEY \
--set minio.secretKey=$$SECRETKEY; \
else \
echo "[ERROR] milvus-minio-secret not found or empty in namespace $(NAMESPACE)"; \
echo " Please ensure Sealed Secrets Controller is running and the secret was decrypted."; \
echo " For local dev: kubectl create secret generic milvus-minio-secret \\"; \
echo " --from-literal=accessKey=<key> --from-literal=secretKey=<key> -n $(NAMESPACE)"; \
exit 1; \
chmod +x scripts/k8s/collect-secrets.sh; \
bash scripts/k8s/collect-secrets.sh --component milvus -n $(NAMESPACE); \
MILVUS_MINIO_ACCESS_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.accesskey}' | base64 -d); \
MILVUS_MINIO_SECRET_KEY=$$(kubectl get secret milvus-minio-secret -n $(NAMESPACE) -o jsonpath='{.data.secretkey}' | base64 -d); \
if [ -f /tmp/datamate-helm-args.sh ]; then \
source /tmp/datamate-helm-args.sh; \
fi; \
helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install \
--set minio.accessKey="$$MILVUS_MINIO_ACCESS_KEY" \
--set minio.secretKey="$$MILVUS_MINIO_SECRET_KEY" \
--set log.persistence.persistentVolumeClaim.accessModes=ReadWriteOnce \
$$HELM_MILVUS_TOLERATIONS; \
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
fi
Expand All @@ -416,13 +426,8 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m
elif [ "$*" = "mineru-310P" ]; then \
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
elif [ "$*" = "datamate" ]; then \
echo ""; \
echo "Remove node configuration (labels/taints)? (y/n) [n]"; \
read -p "> " CLEANUP_NODES; \
if [ "$$CLEANUP_NODES" = "y" ] || [ "$$CLEANUP_NODES" = "Y" ]; then \
$(MAKE) node-cleanup; \
fi; \
helm uninstall datamate -n $(NAMESPACE) --ignore-not-found; \
$(MAKE) node-cleanup; \
elif [ "$*" = "deer-flow" ]; then \
helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found; \
elif [ "$*" = "milvus" ]; then \
Expand Down
10 changes: 7 additions & 3 deletions deployment/helm/datamate/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,16 @@ public:
database: 1Gi
operator: 1Gi
secrets:
create: false # Managed by SealedSecret (deployment/kubernetes/sealed-secrets/)
# Set to false when using Sealed Secrets (managed by install script)
create: false
data:
DB_PASSWORD: "" # Set via secrets.yaml or --set
CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys
DB_PASSWORD: "" # Set via install script or --set
CERT_PASS: "" # Set via install script for encrypted SSL keys
DOMAIN: ""
HOME_PAGE_URL: ""
JWT_SECRET: "" # Auto-generated by install script
LABEL_STUDIO_PASSWORD: "" # Set via install script
LABEL_STUDIO_USER_TOKEN: "" # Auto-generated by install script

datasetVolume: &datasetVolume
name: dataset-volume
Expand Down
12 changes: 12 additions & 0 deletions deployment/helm/label-studio/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,18 @@ spec:
mountPath: /label-studio/local
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
- name: data
persistentVolumeClaim:
Expand Down
20 changes: 10 additions & 10 deletions deployment/helm/milvus/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@ fullnameOverride: ""

## Define toleration for node isolation
## This anchor can be referenced throughout the configuration
nodeIsolationTolerations: &nodeIsolationTolerations
- key: "node-role.kubernetes.io/datamate"
operator: "Equal"
value: "true"
effect: "NoSchedule"
## NOTE: Default is empty array - tolerations should be set via Helm --set
## during install if node isolation is configured
nodeIsolationTolerations: &nodeIsolationTolerations []

## Enable or disable Milvus Cluster mode
cluster:
Expand Down Expand Up @@ -39,7 +37,9 @@ nodeSelector: {}
# Global tolerations
# If set, this will apply to all milvus components
# Individual components can be set to a different tolerations
tolerations: *nodeIsolationTolerations
# Default: empty (no tolerations) - allows scheduling on any node
# Set via --set tolerations[0].key=... during install if node isolation is needed
tolerations: []

# Global affinity
# If set, this will apply to all milvus components
Expand Down Expand Up @@ -218,7 +218,7 @@ log:
## ReadWriteMany access mode required for milvus cluster.
##
storageClass:
accessModes: ReadWriteMany
accessModes: ReadWriteOnce
size: 10Gi
subPath: ""
storagePath:
Expand Down Expand Up @@ -616,7 +616,7 @@ attu:
# - secretName: chart-attu-tls
# hosts:
# - milvus-attu.local

route:
enabled: false
host: ""
Expand Down Expand Up @@ -644,7 +644,7 @@ minio:
pullPolicy: IfNotPresent
accessKey: "" # Set via secrets.yaml or --set
secretKey: "" # Set via secrets.yaml or --set
existingSecret: ""
existingSecret: milvus-minio-secret
bucketName: "milvus-bucket"
rootPath: file
useIAM: false
Expand Down Expand Up @@ -1296,7 +1296,7 @@ kafka:
zookeeper:
enabled: true
replicaCount: 3
image:
image:
repository: bitnamilegacy/zookeeper
tag: 3.7.0

Expand Down
27 changes: 0 additions & 27 deletions deployment/kubernetes/sealed-secrets/datamate.yaml

This file was deleted.

23 changes: 0 additions & 23 deletions deployment/kubernetes/sealed-secrets/label-studio.yaml

This file was deleted.

22 changes: 0 additions & 22 deletions deployment/kubernetes/sealed-secrets/milvus.yaml

This file was deleted.

Loading
Loading