Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 78 additions & 13 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ env:
# Override this repository variable when the stable trunk changes.
STABLE_BRANCH: ${{ vars.STABLE_BRANCH || 'stable/v0.13' }}
MAIN_DOCKER_TAG_SUFFIX: ${{ vars.MAIN_DOCKER_TAG_SUFFIX || '-rc' }}
DOCKER_PYTHON_VERSION: '3.13'
GOTESTCMD: "go test -timeout 1200s -v ./..."
GOPRIVATE: github.com/stackql/*
GH_ACCESS_TOKEN: ${{ secrets.ACTIONS_PRIVATE_PACKAGE_SECRET }}
Expand Down Expand Up @@ -1766,12 +1767,42 @@ jobs:
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.STACKQL_IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true

# Reuses the builder/integration layers from the standard build above.
- name: Build Python runtime image with buildx
uses: docker/build-push-action@v7.4.0
id: img_python_build
if: env.BUILD_IMAGE_REQUIRED == 'true'
with:
context: .
build-args: |
BUILDMAJORVERSION=${{env.BUILDMAJORVERSION}}
BUILDMINORVERSION=${{env.BUILDMINORVERSION}}
BUILDPATCHVERSION=${{env.BUILDPATCHVERSION}}
RUNTIME_IMAGE=python:${{ env.DOCKER_PYTHON_VERSION }}-slim-trixie
platforms: ${{ matrix.platform }}
target: app
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.STACKQL_IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true

- name: Verify Python runtime image
if: env.BUILD_IMAGE_REQUIRED == 'true'
run: |
docker run --rm --entrypoint /bin/sh \
"${{ env.STACKQL_IMAGE_NAME }}@${{ steps.img_python_build.outputs.digest }}" -ec '
python3 -c "import sys; assert sys.version.startswith(sys.argv[1])" "${{ env.DOCKER_PYTHON_VERSION }}."
python3 -m pip --version
stackql --version
stackql_mcp_client --help >/dev/null
'

- name: Export digest
if: env.BUILD_IMAGE_REQUIRED == 'true'
run: |
mkdir -p ${{ runner.temp }}/digests
mkdir -p ${{ runner.temp }}/digests ${{ runner.temp }}/python-digests
digest="${{ steps.img_build.outputs.digest }}"
touch "${{ runner.temp }}/digests/${digest#sha256:}"
touch "${{ runner.temp }}/digests/${digest#sha256:}"
python_digest="${{ steps.img_python_build.outputs.digest }}"
touch "${{ runner.temp }}/python-digests/${python_digest#sha256:}"

- name: Upload digest
if: env.BUILD_IMAGE_REQUIRED == 'true'
Expand All @@ -1781,7 +1812,14 @@ jobs:
path: ${{ runner.temp }}/digests/*
if-no-files-found: error


- name: Upload Python digest
if: env.BUILD_IMAGE_REQUIRED == 'true'
uses: actions/upload-artifact@v7.0.1
with:
name: digests-python-${{ env.PLATFORM_PAIR }}
path: ${{ runner.temp }}/python-digests/*
if-no-files-found: error

dockertest:
env:
IS_DOCKER: 'true'
Expand Down Expand Up @@ -1826,6 +1864,16 @@ jobs:
platform: linux/arm64
db_backend: sqlite
run_integration: false
- job_name: Docker Python Test (linux/amd64, sqlite)
platform: linux/amd64
db_backend: sqlite
image_variant: python
run_integration: false
- job_name: Docker Python Test (linux/arm64, sqlite)
platform: linux/arm64
db_backend: sqlite
image_variant: python
run_integration: false
steps:
- name: Prepare
run: |
Expand Down Expand Up @@ -1883,7 +1931,7 @@ jobs:
if: env.BUILD_IMAGE_REQUIRED == 'true'
with:
path: ${{ runner.temp }}/digests
name: digests-${{ env.PLATFORM_PAIR }}
name: digests-${{ matrix.image_variant == 'python' && 'python-' || '' }}${{ env.PLATFORM_PAIR }}

- name: Prepare digest information
if: env.BUILD_IMAGE_REQUIRED == 'true'
Expand Down Expand Up @@ -2036,13 +2084,13 @@ jobs:
cat ./test/robot/reports/output.xml

# Consumed by dockertestverify, which asserts that the shards of each
# (platform, db backend) group together ran every functional test.
# (platform, db backend, image variant) group together ran every functional test.
# Unsharded legs upload as shard 1 of 1.
- name: Upload robot output for shard verification
if: always() && env.CI_IS_EXPRESS != 'true' && env.BUILD_IMAGE_REQUIRED == 'true'
uses: actions/upload-artifact@v7.0.1
with:
name: robot-output-${{ env.PLATFORM_PAIR }}-${{ matrix.db_backend }}-shard-${{ matrix.robot_shard_index || 1 }}-of-${{ matrix.robot_shard_count || 1 }}
name: robot-output-${{ env.PLATFORM_PAIR }}-${{ matrix.db_backend }}${{ matrix.image_variant == 'python' && '-python' || '' }}-shard-${{ matrix.robot_shard_index || 1 }}-of-${{ matrix.robot_shard_count || 1 }}
path: test/robot/reports/output.xml
if-no-files-found: error

Expand All @@ -2064,7 +2112,7 @@ jobs:

## Sharding splits one robot run across several dockertest legs, so no single
## leg can attest that the whole suite ran. This job runs once all legs have
## concluded and fails unless, for each (platform, db backend) group, the
## concluded and fails unless, for each (platform, db backend, image variant) group, the
## shards together account for every functional test exactly once: both the
## total count and each individual test are checked against the test source,
## parsed independently of the shard modifier. Anything that prevents the
Expand Down Expand Up @@ -2122,10 +2170,17 @@ jobs:
--reports-dir "${{ runner.temp }}/robot-outputs" \
--expected-group linux-amd64-sqlite \
--expected-group linux-amd64-postgres_tcp \
--expected-group linux-arm64-sqlite
--expected-group linux-arm64-sqlite \
--expected-group linux-amd64-sqlite-python \
--expected-group linux-arm64-sqlite-python

dockermerge:
name: Docker merge (${{ matrix.variant }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
variant: [standard, python]
needs:
- dockerbuild
- dockertest
Expand Down Expand Up @@ -2170,7 +2225,7 @@ jobs:
if: env.PUSH_IMAGE_REQUIRED == 'true'
with:
path: ${{ runner.temp }}/digests
pattern: digests-*
pattern: ${{ matrix.variant == 'python' && 'digests-python-*' || 'digests-linux-*' }}
merge-multiple: true

- name: Login to Docker Hub
Expand Down Expand Up @@ -2239,7 +2294,7 @@ jobs:
# Apply the testing suffix to every main tag in one place.
flavor: |
latest=false
suffix=${{ github.ref_type == 'branch' && github.ref_name == 'main' && env.MAIN_DOCKER_TAG_SUFFIX || '' }}
suffix=${{ matrix.variant == 'python' && format('-python{0}', env.DOCKER_PYTHON_VERSION) || '' }}${{ github.ref_type == 'branch' && github.ref_name == 'main' && env.MAIN_DOCKER_TAG_SUFFIX || '' }}
tags: |
type=ref,event=branch
type=ref,event=pr
Expand All @@ -2251,10 +2306,20 @@ jobs:
- name: Create manifest list and push
if: env.PUSH_IMAGE_REQUIRED == 'true'
working-directory: ${{ runner.temp }}/digests
env:
PYTHON_SUFFIX: ${{ matrix.variant == 'python' && format('-python{0}', env.DOCKER_PYTHON_VERSION) || '' }}${{ matrix.variant == 'python' && github.ref_type == 'branch' && github.ref_name == 'main' && env.MAIN_DOCKER_TAG_SUFFIX || '' }}
PYTHON_ALIAS: ${{ matrix.variant == 'python' && '-python3' || '' }}${{ matrix.variant == 'python' && github.ref_type == 'branch' && github.ref_name == 'main' && env.MAIN_DOCKER_TAG_SUFFIX || '' }}
run: |
# shellcheck disable=SC2046
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.STACKQL_IMAGE_NAME }}@sha256:%s ' *)
# Both Python tag forms are attached to the same multi-platform manifest.
mapfile -t tags < <(jq -r --arg suffix "$PYTHON_SUFFIX" --arg alias "$PYTHON_ALIAS" '
.tags | if $suffix != "" then . + map(rtrimstr($suffix) + $alias) else . end
| unique | .[] | "-t", .
' <<< "$DOCKER_METADATA_OUTPUT_JSON")
images=()
for digest in *; do
images+=("${STACKQL_IMAGE_NAME}@sha256:${digest}")
done
docker buildx imagetools create "${tags[@]}" "${images[@]}"

- name: Inspect image
if: env.PUSH_IMAGE_REQUIRED == 'true'
Expand Down
18 changes: 15 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@

ARG RUNTIME_IMAGE=ubuntu:26.04

FROM golang:1.27.3-bookworm AS sourceprep

ENV SRC_DIR=/work/stackql/src
Expand Down Expand Up @@ -150,7 +153,7 @@ COPY --from=builder /work/stackql/build/stackql_mcp_client ${TEST_ROOT_DIR}/buil

RUN if [ "${RUN_INTEGRATION_TESTS}" = "1" ]; then env PYTHONPATH="$PYTHONPATH:${TEST_ROOT_DIR}/test/python" robot ${TEST_ROOT_DIR}/test/robot/functional; fi

FROM ubuntu:22.04 AS app
FROM ${RUNTIME_IMAGE} AS app

ARG TEST_ROOT_DIR=/opt/test/stackql

Expand All @@ -175,8 +178,12 @@ ENV STACKQL_CFG_ROOT="${STACKQL_CFG_ROOT}"

ENV STACKQL_PG_PORT="${STACKQL_PG_PORT}"

RUN groupadd --gid ${STACKQL_GID} stackql \
&& useradd --uid ${STACKQL_UID} --gid ${STACKQL_GID} --home-dir ${STACKQL_CFG_ROOT} --no-create-home --shell /bin/bash stackql \
RUN if getent group "${STACKQL_GID}" >/dev/null; then \
groupmod --new-name stackql "$(getent group "${STACKQL_GID}" | cut -d: -f1)"; \
else groupadd --gid "${STACKQL_GID}" stackql; fi \
&& if getent passwd "${STACKQL_UID}" >/dev/null; then \
usermod --login stackql --gid "${STACKQL_GID}" --home "${STACKQL_CFG_ROOT}" --shell /bin/bash "$(getent passwd "${STACKQL_UID}" | cut -d: -f1)"; \
else useradd --uid "${STACKQL_UID}" --gid "${STACKQL_GID}" --home-dir "${STACKQL_CFG_ROOT}" --no-create-home --shell /bin/bash stackql; fi \
&& mkdir -p ${APP_DIR} ${STACKQL_CFG_ROOT}/keys ${STACKQL_CFG_ROOT}/srv/credentials ${STACKQL_CFG_ROOT}/credentials/dummy ${STACKQL_CFG_ROOT}/registry ${STACKQL_CFG_ROOT}/logs ${STACKQL_CFG_ROOT}/db \
&& chown -R ${STACKQL_UID}:${STACKQL_GID} ${STACKQL_CFG_ROOT}

Expand All @@ -192,6 +199,11 @@ RUN apt-get update \

EXPOSE ${STACKQL_PG_PORT}/tcp

RUN --mount=type=bind,source=cicd/docker/py-img-requirements.txt,target=/tmp/py-img-requirements.txt \
if command -v python3 >/dev/null 2>&1; then \
python3 -m pip install --no-cache-dir -r /tmp/py-img-requirements.txt; \
fi

USER ${STACKQL_UID}:${STACKQL_GID}

WORKDIR ${STACKQL_CFG_ROOT}
Expand Down
1 change: 1 addition & 0 deletions cicd/docker/py-img-requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
PyYAML==6.0.3
Loading