ROB-928 Harden runner pod security: non-root uid 1000, read-only root FS, dropped capabilities

Dockerfile

@@ -86,7 +86,8 @@ RUN apt-get update \
 
 
 # Patching CVE-2024-32002
-RUN git config --global core.symlinks false
+# --system rather than --global so the setting also applies to the non-root runtime user
+RUN git config --system core.symlinks false
 
 # Temporary setuptools CVE fix untill python:3.12-slim image will be used.
 RUN rm -rf /usr/local/lib/python3.11/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl
@@ -116,6 +117,16 @@ RUN chmod 0644 /etc/apt/keyrings/kubernetes-apt-keyring.asc \
     && apt-get install -y --no-install-recommends kubectl \
     && rm -rf /var/lib/apt/lists/*
 
+# Run as a non-root user. uid/gid 1000 matches runner.securityContext.pod in the Helm chart.
+RUN groupadd --gid 1000 robusta \
+    && useradd --uid 1000 --gid 1000 --create-home --home-dir /home/robusta --shell /sbin/nologin robusta \
+    && mkdir -p /home/robusta/.ssh /home/robusta/.cache \
+    && chmod 0700 /home/robusta/.ssh \
+    && chown -R 1000:1000 /app /venv /etc/robusta /home/robusta
+
+ENV HOME=/home/robusta
+USER 1000
+
 # Run the application
 # -u disables stdout buffering https://stackoverflow.com/questions/107705/disable-output-buffering
 CMD [ "python3", "-u", "-m", "robusta.runner.main"]

docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst

@@ -165,6 +165,15 @@ install command for the package being installed will be run with `--no-build-iso
 the `pip docs <https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-no-build-isolation>`_
 for details).
 
+Read-Only Filesystem Limitation
+*********************************
+
+The runner runs with a read-only root filesystem by default (see the ``runner.hardenedFs`` Helm value).
+Runtime installs into Python's ``site-packages`` still work, but a package that declares
+``console_scripts`` entry points cannot be pip-installed at runtime, because the scripts target the
+read-only ``/venv/bin``. For such packages, either set ``runner.hardenedFs: false``, or bake the package
+into a custom image as described below.
+
 Baking Actions into a Custom Image
 --------------------------------------
 

docs/setup-robusta/privacy-and-security.rst

@@ -18,6 +18,23 @@ Handling Secrets in Robusta's Helm Values
 ******************************************
 Refer to :ref:`Managing Secrets`.
 
+Runner Pod Security
+******************************************
+
+By default, the Robusta runner pod runs as a non-root user (uid 1000) with all Linux capabilities dropped,
+``seccompProfile: RuntimeDefault`` and privilege escalation disabled.
+
+The runner container's root filesystem is also mounted read-only, with writable ``emptyDir`` volumes only
+where the runner needs to write at runtime (``/tmp``, git playbook clones, pip caches and runtime-installed
+Python packages). This is controlled by the ``runner.hardenedFs`` Helm value (default ``true``). Setting it
+to ``false`` keeps a writable root filesystem while still running as non-root. An explicit
+``runner.securityContext.container.readOnlyRootFilesystem`` value always takes precedence over ``hardenedFs``.
+
+.. note::
+
+    With the read-only filesystem enabled, external playbook packages that declare ``console_scripts``
+    entry points cannot be pip-installed at runtime. Refer to :ref:`Loading External Actions`.
+
 Limiting Robusta's Access in Your Cluster
 *******************************************
 

helm/robusta/templates/runner.yaml

@@ -77,12 +77,17 @@ spec:
         {{ else }}
         image: {{ .Values.image.registry }}/{{ .Values.runner.imageName }}
         {{- end }}
+        # Copies the baked site-packages into the emptyDir that shadows it, so runtime playbook
+        # pip installs have somewhere to write. Copies the entries rather than "cp -a ${SRC}/."
+        # because that form also tries to preserve timestamps on the destination directory, which
+        # the non-root user does not own - cp then exits 1 and the whole pod fails to start.
         command:
           - sh
           - -c
           - >
             SRC="/venv/lib/python$(python -V | cut -d' ' -f2 | cut -d. -f1,2)/site-packages" &&
-            cp -a "${SRC}/." /venv-writable/
+            cd "${SRC}" &&
+            find . -mindepth 1 -maxdepth 1 -exec cp -a -t /venv-writable {} +
         volumeMounts:
         - name: venv-lib-volume
           mountPath: /venv-writable
@@ -95,16 +100,24 @@ spec:
         image: {{ .Values.image.registry }}/{{ .Values.runner.imageName }}
         {{- end }}
         imagePullPolicy: {{ .Values.runner.imagePullPolicy }}
-        {{- with .Values.runner.securityContext.container }}
-        securityContext:
-        {{- if $.Values.runner.hardenedFs }}
-          {{- $hardened := merge (dict "readOnlyRootFilesystem" true) . }}
-          {{- toYaml $hardened | nindent 12 }}
-        {{- else }}
-          {{- toYaml . | nindent 12 }}
+        {{- $containerSecurityContext := deepCopy (default dict .Values.runner.securityContext.container) }}
+        {{- /* hardenedFs provides the writable mounts that a read-only root filesystem needs, so
+               the two are always turned on together. hasKey rather than merge: sprig's merge
+               treats an explicit "false" as an absent value and would silently override it. */}}
+        {{- if and .Values.runner.hardenedFs (not (hasKey $containerSecurityContext "readOnlyRootFilesystem")) }}
+        {{- $containerSecurityContext = set $containerSecurityContext "readOnlyRootFilesystem" true }}
         {{- end }}
+        {{- if $containerSecurityContext }}
+        securityContext:
+          {{- toYaml $containerSecurityContext | nindent 12 }}
         {{- end }}
         env:
+          # kubectl and CPython both write caches relative to $HOME / the source tree, which are
+          # on the read-only root filesystem. Redirect them to the writable /tmp mount.
+          - name: KUBECACHEDIR
+            value: /tmp/.kube-cache
+          - name: PYTHONPYCACHEPREFIX
+            value: /tmp/pycache
           - name: PLAYBOOKS_CONFIG_FILE_PATH
             value: /etc/robusta/config/active_playbooks.yaml
           - name: RELEASE_NAME
@@ -190,7 +203,9 @@ spec:
           - name: app-git-volume
             mountPath: /app/robusta-git
           - name: cache-volume
-            mountPath: /root/.cache
+            mountPath: /home/robusta/.cache
+          - name: ssh-volume
+            mountPath: /home/robusta/.ssh
           - name: venv-lib-volume
             mountPath: /venv/lib/python3.11/site-packages
           {{- end }}
@@ -254,6 +269,8 @@ spec:
           emptyDir: {}
         - name: cache-volume
           emptyDir: {}
+        - name: ssh-volume
+          emptyDir: {}
         - name: venv-lib-volume
           emptyDir: {}
         {{- end }}

helm/robusta/values.yaml

@@ -711,6 +711,9 @@ grafanaRenderer:
       memory: 512Mi
     limits:
       cpu: ~
+  # This container runs inside the runner pod, so it inherits runner.securityContext.pod -
+  # including runAsNonRoot/runAsUser 1000. If this third-party image needs a different uid, or
+  # declares USER root, override runAsNonRoot/runAsUser here when enabling it.
   securityContext:
     container:
       privileged: false
@@ -766,12 +769,27 @@ runner:
   securityContext:
     container:
       allowPrivilegeEscalation: false
-      capabilities: {}
+      capabilities:
+        drop:
+          - ALL
       privileged: false
-      readOnlyRootFilesystem: false
-    pod: {}
-  # Enable hardened filesystem security (read-only root filesystem with writable volume mounts)
-  hardenedFs: false
+      # readOnlyRootFilesystem is intentionally NOT set here - it is derived from hardenedFs
+      # below, so that the read-only root FS always comes together with the writable mounts it
+      # needs. Set it explicitly only if you know what you are doing; an explicit value wins.
+    pod:
+      runAsNonRoot: true
+      runAsUser: 1000
+      runAsGroup: 1000
+      # emptyDir/PVC mounts are created root-owned; fsGroup makes them writable for uid 1000
+      fsGroup: 1000
+      seccompProfile:
+        type: RuntimeDefault
+  # Enable hardened filesystem security: adds readOnlyRootFilesystem to the runner container
+  # plus the writable emptyDir mounts the runner needs at runtime (/tmp, the git clone dir, the
+  # pip/HOME cache, ~/.ssh and site-packages for runtime playbook installs).
+  # Known limitation: /venv/bin stays read-only, so a playbook package that declares
+  # console_scripts entry points cannot be pip-installed at runtime while this is enabled.
+  hardenedFs: true
   setKRRSecurityContext: false
   #Enabled custom DNS configuration for runner
   dnsConfig:

src/robusta/integrations/git/git_repo.py

@@ -14,7 +14,10 @@
 
 GIT_DIR_NAME = "robusta-git"
 REPO_LOCAL_BASE_DIR = os.path.abspath(os.path.join(os.environ.get("REPO_LOCAL_BASE_DIR", "/app"), GIT_DIR_NAME))
-SSH_ROOT_DIR = os.environ.get("SSH_ROOT_DIR", "/root/.ssh")
+# The runner does not necessarily run as root, so default to the current user's home
+# rather than a hardcoded /root. Must stay in sync with where ssh looks for known_hosts
+# (see GIT_SSH_COMMAND below).
+SSH_ROOT_DIR = os.environ.get("SSH_ROOT_DIR", os.path.join(os.environ.get("HOME", "/root"), ".ssh"))
 GIT_REPOS_VERIFIED_HOSTS = os.environ.get("GIT_REPOS_VERIFIED_HOSTS", "")
 
 GIT_SSH_PREFIX = "git@"
@@ -33,8 +36,7 @@ def setup_host_keys(cls, custom_host_keys: List[str]):
         if cls.host_keys_initialized:
             return
 
-        if not os.path.exists(SSH_ROOT_DIR):
-            os.mkdir(SSH_ROOT_DIR)
+        os.makedirs(SSH_ROOT_DIR, exist_ok=True)
         with open(f"{SSH_ROOT_DIR}/known_hosts", "w") as f:
             for key in WELL_KNOWN_HOST_KEYS + custom_host_keys:
                 key = key.strip()
@@ -89,7 +91,12 @@ def __init__(self, git_repo_url: str, git_key: str, git_branch: str = None):
         else:
             ssh_key_option = ""
 
-        self.env["GIT_SSH_COMMAND"] = f"ssh {ssh_key_option} -o IdentitiesOnly=yes"
+        # Point ssh at the known_hosts file setup_host_keys() actually writes. Without this,
+        # ssh reads $HOME/.ssh/known_hosts, which is only the same file when SSH_ROOT_DIR
+        # happens to sit under $HOME.
+        self.env["GIT_SSH_COMMAND"] = (
+            f"ssh {ssh_key_option} -o IdentitiesOnly=yes -o UserKnownHostsFile={SSH_ROOT_DIR}/known_hosts"
+        )
         self.repo_lock = threading.RLock()
         self.repo_name = os.path.splitext(os.path.basename(git_repo_url))[0]
         self.repo_local_path = os.path.join(REPO_LOCAL_BASE_DIR, self.repo_name)

tests/test_helm_chart.py

@@ -0,0 +1,112 @@
+import os
+import re
+
+import yaml
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CHART_DIR = os.path.join(REPO_ROOT, "helm", "robusta")
+RUNNER_TEMPLATE = os.path.join(CHART_DIR, "templates", "runner.yaml")
+DOCKERFILE = os.path.join(REPO_ROOT, "Dockerfile")
+
+
+def _values() -> dict:
+    with open(os.path.join(CHART_DIR, "values.yaml")) as f:
+        return yaml.safe_load(f)
+
+
+def _runner_template() -> str:
+    with open(RUNNER_TEMPLATE) as f:
+        return f.read()
+
+
+def _dockerfile() -> str:
+    with open(DOCKERFILE) as f:
+        return f.read()
+
+
+def test_runner_pod_hardened_defaults():
+    runner = _values()["runner"]
+    pod = runner["securityContext"]["pod"]
+    container = runner["securityContext"]["container"]
+
+    assert pod["runAsNonRoot"] is True
+    assert pod["runAsUser"] == 1000
+    assert pod["runAsGroup"] == 1000
+    assert pod["fsGroup"] == 1000
+    assert pod["seccompProfile"] == {"type": "RuntimeDefault"}
+
+    assert container["capabilities"] == {"drop": ["ALL"]}
+    assert container["allowPrivilegeEscalation"] is False
+    assert container["privileged"] is False
+
+    # the read-only root filesystem needs the writable emptyDir mounts that hardenedFs adds
+    assert runner["hardenedFs"] is True
+
+
+def test_readonly_root_fs_is_derived_from_hardened_fs():
+    # readOnlyRootFilesystem must NOT be a standalone default: hardenedFs is the single switch,
+    # so that turning hardenedFs off can never leave a read-only FS without writable mounts.
+    container = _values()["runner"]["securityContext"]["container"]
+    assert "readOnlyRootFilesystem" not in container
+
+    template = _runner_template()
+    assert '"readOnlyRootFilesystem" true' in template
+    assert ".Values.runner.hardenedFs" in template
+
+
+def test_runner_image_runs_as_non_root_user():
+    directives = [line.strip() for line in _dockerfile().splitlines() if line.strip().startswith("USER ")]
+
+    # the last USER directive is what the container actually runs as
+    assert directives, "Dockerfile must set a USER so the runner does not run as root"
+    assert directives[-1] == "USER 1000"
+
+
+def test_hardened_mounts_cover_runtime_write_paths():
+    template = _runner_template()
+    mount_paths = set(re.findall(r"^\s*mountPath:\s*(\S+)\s*$", template, re.MULTILINE))
+
+    # every directory the runner writes to at runtime must be a writable mount once the root
+    # filesystem is read-only: /tmp (tempfiles, certs), the git clone dir, the pip/HOME cache,
+    # ~/.ssh (known_hosts for git@ repos) and site-packages (runtime playbook pip installs)
+    for path in ("/tmp", "/app/robusta-git", "/home/robusta/.cache", "/home/robusta/.ssh"):
+        assert path in mount_paths, f"{path} is written at runtime but is not a mount"
+    assert any(p.endswith("/site-packages") for p in mount_paths)
+
+
+def test_venv_mount_matches_dockerfile_python_version():
+    # The site-packages mountPath is version-pinned while the setup-venv initContainer derives
+    # the version at runtime. If the base image's python is bumped without updating the mount,
+    # runtime pip installs would silently target the read-only image layer.
+    final_stage_images = re.findall(r"^FROM\s+python:(\d+\.\d+)", _dockerfile(), re.MULTILINE)
+    assert final_stage_images, "could not determine the python version from the Dockerfile"
+    python_version = final_stage_images[-1]
+
+    mount_paths = re.findall(r"^\s*mountPath:\s*(\S*site-packages)\s*$", _runner_template(), re.MULTILINE)
+    assert mount_paths, "no site-packages mountPath found in the runner template"
+    for path in mount_paths:
+        assert f"python{python_version}" in path, (
+            f"site-packages mount {path} does not match Dockerfile python {python_version}"
+        )
+
+
+def test_home_matches_cache_and_ssh_mounts():
+    # pip's cache and ssh's known_hosts are resolved via $HOME by child processes, so the
+    # Dockerfile's HOME and the chart's mountPaths have to stay in sync.
+    home = re.search(r"^ENV HOME=(\S+)\s*$", _dockerfile(), re.MULTILINE)
+    assert home, "Dockerfile must set HOME explicitly for the non-root user"
+    mount_paths = set(re.findall(r"^\s*mountPath:\s*(\S+)\s*$", _runner_template(), re.MULTILINE))
+    assert f"{home.group(1)}/.cache" in mount_paths
+    assert f"{home.group(1)}/.ssh" in mount_paths
+
+def test_venv_init_container_copy_works_as_non_root():
+    # "cp -a <src>/. <dst>/" also applies the source directory's attributes to the destination
+    # directory, which the non-root user does not own: cp exits 1, the init container fails and
+    # the pod never starts. The copy must therefore operate on the entries, not on ".".
+    template = _runner_template()
+    init_copy = [line.strip() for line in template.splitlines() if "/venv-writable" in line and "cp" in line]
+    assert init_copy, "no copy command targeting /venv-writable found"
+    for line in init_copy:
+        assert '/." /venv-writable' not in line, (
+            f"copy form exits 1 as non-root because it preserves attrs on the destination dir: {line}"
+        )