diff --git a/.bumpversion.toml b/.bumpversion.toml index f2a79f6c7..0090ed5b3 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -12,3 +12,23 @@ replace = '__version__ = "{new_version}"' filename = "pyproject.toml" search = 'version = "{current_version}"' replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "Helm/Chart.yaml" +search = 'version: {current_version}' +replace = 'version: {new_version}' + +[[tool.bumpversion.files]] +filename = "Helm/charts/murfey-instrument-server-clem/Chart.yaml" +search = 'version: {current_version}' +replace = 'version: {new_version}' + +[[tool.bumpversion.files]] +filename = "Helm/charts/murfey-rsync/Chart.yaml" +search = 'version: {current_version}' +replace = 'version: {new_version}' + +[[tool.bumpversion.files]] +filename = "Helm/charts/murfey-server/Chart.yaml" +search = 'version: {current_version}' +replace = 'version: {new_version}' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00200fe5a..04d777267 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,7 @@ repos: name: Verifying JSON file syntax - id: check-yaml name: Verifying YAML file syntax + exclude: Helm/charts/.*?/templates/deployment\.yaml args: [--allow-multiple-documents] - id: check-toml name: Verifying TOML file syntax @@ -103,4 +104,5 @@ repos: hooks: - id: prettier name: Formatting other file types according to Prettier + exclude: Helm/charts/.*?/templates/deployment\.yaml # types_or: [css, html, json, yaml] diff --git a/Dockerfiles/murfey-instrument-server b/Dockerfiles/murfey-instrument-server new file mode 100644 index 000000000..dcd32d45f --- /dev/null +++ b/Dockerfiles/murfey-instrument-server @@ -0,0 +1,53 @@ +# Build command template +# podman build --build-arg groupid= --build-arg userid= --build-arg groupname= --no-cache -f path/to/Dockerfiles/murfey-instrument-server -t murfey-instrument-server: path/to/python-murfey + +# Set up the base image to build with +FROM docker.io/library/python:3.12.8-slim-bullseye AS base + +# Install Vim in base image +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + rsync \ + vim \ + && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + + +# Build Murfey in a branch image +FROM base as build +COPY ./ /python-murfey/ +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + busybox \ + net-tools \ + libpq-dev \ + && \ + busybox --install && \ + python -m venv /venv && \ + /venv/bin/python -m pip install --upgrade \ + pip \ + build \ + importlib-metadata && \ + /venv/bin/python -m pip install /python-murfey[client,instrument-server] + + +# Transfer completed Murfey build to base image +FROM base + +# Define external build arguments +ARG groupid +ARG groupname +ARG userid + +# Copy completed Murfey build across and set user and group permissions +COPY --from=build /venv/ /venv/ +RUN groupadd -r -g "${groupid}" "${groupname}" && \ + useradd -r -M "${groupname}" -u "${userid}" -g "${groupname}" && \ + chown -R "${userid}":"${groupid}" /venv && \ + chmod -R a+x /venv +ENV PATH=/venv/bin:$PATH +USER "${userid}":"${groupid}" diff --git a/Dockerfiles/murfey-rsync b/Dockerfiles/murfey-rsync new file mode 100644 index 000000000..77ef74e27 --- /dev/null +++ b/Dockerfiles/murfey-rsync @@ -0,0 +1,14 @@ +# Template build command +# podman build --build-arg groupid= --build-arg userid= --build-arg groupname= --no-cache -f path/to/Dockerfiles/murfey-rsync + +FROM docker.io/library/alpine:3.20 +# FROM alpine:3.14 + +ARG groupid +ARG groupname +ARG userid + +# Add any system dependencies for the developer/build environment here +RUN apk add --no-cache rsync && \ + addgroup -S -g "${groupid}" "${groupname}" && \ + adduser -S "${groupname}" -G "${groupname}" -u "${userid}" -s /bin/sh diff --git a/Dockerfiles/murfey-server b/Dockerfiles/murfey-server new file mode 100644 index 000000000..bf2b30ce8 --- /dev/null +++ b/Dockerfiles/murfey-server @@ -0,0 +1,61 @@ +# Template build command +# podman build --build-arg groupid= --build-arg userid= --build-arg groupname= --no-cache -f path/to/Dockerfiles/murfey-server -t murfey-server: path/to/python-murfey + +# Set up the base image to build with +FROM docker.io/library/python:3.12.8-slim-bullseye AS base + +# Install Vim and PostgreSQL dependencies in base image +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + libpq-dev \ + vim \ + && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + + +# Build Murfey and IMOD in a branch image +FROM base as build +COPY ./ /python-murfey/ +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + busybox \ + curl \ + net-tools \ + && \ + busybox --install && \ + python -m venv /venv && \ + /venv/bin/python -m pip install --upgrade \ + pip \ + build \ + importlib-metadata \ + psycopg2-binary \ + && \ + /venv/bin/python -m pip install /python-murfey[server] && \ + curl https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_5.1.0_RHEL8-64_CUDA12.0.sh > imod_5.1.0_RHEL8-64_CUDA12.0.sh && \ + chmod +x imod_5.1.0_RHEL8-64_CUDA12.0.sh && \ + mkdir imod && \ + ./imod_5.1.0_RHEL8-64_CUDA12.0.sh -dir imod -skip -y + + +# Transfer completed builds to base image +FROM base + +# Pass external build arguments to this stage +ARG groupid +ARG groupname +ARG userid + +# Copy completed Murfey and IMOD builds across and set user and group permissions +COPY --from=build /venv/ /venv/ +COPY --from=build /imod/ /imod/ +RUN groupadd -r -g "${groupid}" "${groupname}" && \ + useradd -r -M "${groupname}" -u "${userid}" -g "${groupname}" && \ + chown -R "${userid}":"${groupid}" /venv && \ + chmod -R a+x /venv +ENV PATH=/venv/bin:/imod/IMOD/bin:$PATH +ENV IMOD_DIR=/imod/IMOD +USER "${userid}":"${groupid}" diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml new file mode 100644 index 000000000..f6ffd63ef --- /dev/null +++ b/Helm/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +name: murfey-services +description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data +version: 0.16.9 +dependencies: + - name: murfey-instrument-server-clem + - name: murfey-server + - name: murfey-rsync diff --git a/Helm/charts/murfey-instrument-server-clem/Chart.yaml b/Helm/charts/murfey-instrument-server-clem/Chart.yaml new file mode 100644 index 000000000..aa304a3e6 --- /dev/null +++ b/Helm/charts/murfey-instrument-server-clem/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: murfey-instrument-server-clem +description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files +version: 0.16.9 diff --git a/Helm/charts/murfey-instrument-server-clem/templates/deployment.yaml b/Helm/charts/murfey-instrument-server-clem/templates/deployment.yaml new file mode 100644 index 000000000..69fae72ef --- /dev/null +++ b/Helm/charts/murfey-instrument-server-clem/templates/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + {{- toYaml .Values.servicePorts | nindent 2 }} + selector: + app: {{ .Values.appName }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ .Values.appName }} + template: + metadata: + labels: + app: {{ .Values.appName }} + spec: + securityContext: + runAsUser: {{ .Values.global.runAsUser }} + runAsGroup: {{ .Values.global.runAsGroup }} + volumes: + # Mount config files from secrets + - name: murfey-client-config + secret: + secretName: {{ .Values.global.murfeyClientConfigCLEMSecretName }} + items: + - key: {{ .Values.global.murfeyClientConfigCLEMFileName }} + path: .murfey + # Mount data directories + {{- toYaml .Values.extraVolumes | nindent 8 }} + containers: + - name: {{ .Values.appName }} + image: {{ .Values.image }} + imagePullPolicy: Always + securityContext: + privileged: false + volumeMounts: + # Mount Murfey client config + - name: murfey-client-config + mountPath: /murfey/config/.murfey + subPath: .murfey + readOnly: false + # Mount data directories + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + env: + - name: MURFEY_CLIENT_CONFIG_HOME + value: "/tmp" + ports: + - containerPort: {{ .Values.containerPort }} + command: + {{- toYaml .Values.command | nindent 12 }} + args: + {{- toYaml .Values.args | nindent 12 }} diff --git a/Helm/charts/murfey-rsync/Chart.yaml b/Helm/charts/murfey-rsync/Chart.yaml new file mode 100644 index 000000000..8babbb199 --- /dev/null +++ b/Helm/charts/murfey-rsync/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: murfey-rsync +description: Helm chart for deploying an rsync daemon, which is responsible for executing the transfer of files from the client storage directory to the server storage system +version: 0.16.9 diff --git a/Helm/charts/murfey-rsync/templates/deployment.yaml b/Helm/charts/murfey-rsync/templates/deployment.yaml new file mode 100644 index 000000000..3bbd29dfb --- /dev/null +++ b/Helm/charts/murfey-rsync/templates/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.appName }} +spec: + ports: + {{- toYaml .Values.servicePorts | nindent 2 }} + selector: + app: {{ .Values.appName }} + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Values.appName }} + template: + metadata: + labels: + app: {{ .Values.appName }} + spec: + securityContext: + runAsUser: {{ .Values.global.runAsUser }} + runAsGroup: {{ .Values.global.runAsGroup }} + volumes: + - name: rsyncd-conf + secret: + secretName: {{ .Values.global.rsyncConfigSecretName }} + items: + - key: {{ .Values.global.rsyncConfigFileName }} + path: rsyncd.conf + # Mount data directories + {{- toYaml .Values.extraVolumes | nindent 8 }} + containers: + - name: {{ .Values.appName }} + image: {{ .Values.image }} + imagePullPolicy: Always + resources: + requests: + cpu: {{ .Values.cpuRequest }} + limits: + cpu: {{ .Values.cpuLimit }} + memory: {{ .Values.memoryLimit }} + volumeMounts: + # Mount rsync config files + - name: rsyncd-conf + mountPath: /murfey/rsync/rsyncd.conf + subPath: rsyncd.conf + readOnly: true + # Mount data directories + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + ports: + - containerPort: {{ .Values.containerPort }} + command: + {{- toYaml .Values.command | nindent 12 }} + args: + {{- toYaml .Values.args | nindent 12 }} diff --git a/Helm/charts/murfey-server/Chart.yaml b/Helm/charts/murfey-server/Chart.yaml new file mode 100644 index 000000000..c2b7ff143 --- /dev/null +++ b/Helm/charts/murfey-server/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: murfey-server +description: Helm chart for deploying a Murfey backend server, which is responsible for orchestrating the data transfer and processing workflow between the client PC and the storage system +version: 0.16.9 diff --git a/Helm/charts/murfey-server/templates/deployment.yaml b/Helm/charts/murfey-server/templates/deployment.yaml new file mode 100644 index 000000000..0901e3c24 --- /dev/null +++ b/Helm/charts/murfey-server/templates/deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + {{- toYaml .Values.servicePorts | nindent 2 }} + selector: + app: {{ .Values.appName }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ .Values.appName }} + template: + metadata: + labels: + app: {{ .Values.appName }} + spec: + securityContext: + runAsUser: {{ .Values.global.runAsUser }} + runAsGroup: {{ .Values.global.runAsGroup }} + volumes: + # Mount config files from secrets + - name: secrets + projected: + sources: + - secret: + name: {{ .Values.global.murfeyMachineConfigSecretName }} + - secret: + name: {{ .Values.global.murfeySecurityConfigSecretName }} + - secret: + name: {{ .Values.global.ispybSecretName }} + - secret: + name: {{ .Values.global.murfeyDBSecretName }} + - secret: + name: {{ .Values.global.rmqSecretName }} + # Mount data directories + {{- toYaml .Values.extraVolumes | nindent 8 }} + containers: + - name: {{ .Values.appName }} + image: {{ .Values.image }} + imagePullPolicy: Always + securityContext: + privileged: false + volumeMounts: + # Mount config files + - name: secrets + mountPath: /murfey/config + readOnly: true + # Mount data directories + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + env: + - name: MURFEY_MACHINE_CONFIGURATION + value: "/murfey/config/{{ .Values.global.murfeyMachineConfigFileName }}" + - name: MURFEY_SECURITY_CONFIGURATION + value: "/murfey/config/{{ .Values.global.murfeySecurityConfigFileName }}" + - name: ISPYB_CREDENTIALS + value: "/murfey/config/{{ .Values.global.ispybFileName }}" + ports: + - containerPort: {{ .Values.containerPort }} + command: + {{- toYaml .Values.command | nindent 12 }} + args: + {{- toYaml .Values.args | nindent 12 }}