Skip to content

perf(images): persist multi-platform build caches #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
imbajin merged 5 commits into from
Jul 12, 2026
Merged

perf(images): persist multi-platform build caches #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1ecac6a
perf(hubble): reuse registry build cache
imbajin Jul 12, 2026
47bb6ef
chore: update checkout action
imbajin Jul 12, 2026
b9f11ae
fix(hubble): isolate build cache channels
imbajin Jul 12, 2026
0b63fce
fix(workflows): default optional cache image
imbajin Jul 12, 2026
0da979f
chore(workflows): speed up multi-arch builds
imbajin Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 34 additions & 9 deletions .github/workflows/_publish_image_reusable.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ jobs:
IMAGE_REPO_RELEASE: ${{ matrix.image_repo_release }}
PLATFORMS_LATEST: ${{ matrix.platforms_latest }}
PLATFORMS_RELEASE: ${{ matrix.platforms_release }}
REGISTRY_CACHE_IMAGE: ${{ matrix.registry_cache_image || '' }}
run: |
set -euo pipefail

Expand All @@ -159,14 +160,38 @@ jobs:
image_url="${image_repo}:${VERSION_TAG}"
cache_scope="${COMPONENT}-${MODULE}"

if [ -n "$REGISTRY_CACHE_IMAGE" ]; then
cache_from="type=registry,ref=${REGISTRY_CACHE_IMAGE}"
cache_to_min="type=registry,ref=${REGISTRY_CACHE_IMAGE},mode=min"
cache_to_max="type=registry,ref=${REGISTRY_CACHE_IMAGE},mode=max"
cache_backend="registry"
else
cache_from="type=gha,scope=${cache_scope}"
cache_to_min="type=gha,scope=${cache_scope},mode=min"
cache_to_max="type=gha,scope=${cache_scope},mode=max"
cache_backend="gha"
fi

{
echo "image_url=$image_url"
echo "platforms=$platforms"
echo "cache_scope=$cache_scope"
echo "cache_from=$cache_from"
echo "cache_to_min=$cache_to_min"
echo "cache_to_max=$cache_to_max"
echo "cache_backend=$cache_backend"
} >> "$GITHUB_OUTPUT"

{
echo "### Build parameters: ${MODULE}"
echo
echo "| Image | Platforms | Cache backend | Cache reference |"
echo "| --- | --- | --- | --- |"
echo "| \`${image_url}\` | \`${platforms}\` | \`${cache_backend}\` | \`${REGISTRY_CACHE_IMAGE:-$cache_scope}\` |"
} >> "$GITHUB_STEP_SUMMARY"

- name: Checkout source (${{ matrix.module }})
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
repository: ${{ inputs.repository_url }}
ref: ${{ needs.prepare.outputs.checkout_ref }}
Expand Down Expand Up @@ -196,8 +221,8 @@ jobs:
platforms: linux/amd64
load: true
tags: ${{ steps.params.outputs.image_url }}
cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }}
cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=min
cache-from: ${{ steps.params.outputs.cache_from }}
cache-to: ${{ steps.params.outputs.cache_to_min }}
build-args: ${{ inputs.mvn_args }}

- name: Build x86 image for smoke test (${{ matrix.module }})
Expand All @@ -209,8 +234,8 @@ jobs:
platforms: linux/amd64
load: true
tags: ${{ steps.params.outputs.image_url }}
cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }}
cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=min
cache-from: ${{ steps.params.outputs.cache_from }}
cache-to: ${{ steps.params.outputs.cache_to_min }}

- name: Run smoke test (${{ matrix.module }})
if: ${{ matrix.smoke_test }}
Expand Down Expand Up @@ -239,8 +264,8 @@ jobs:
platforms: ${{ steps.params.outputs.platforms }}
push: true
tags: ${{ steps.params.outputs.image_url }}
cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }}
cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=max
cache-from: ${{ steps.params.outputs.cache_from }}
cache-to: ${{ steps.params.outputs.cache_to_max }}
build-args: ${{ inputs.mvn_args }}

- name: Build and push image (${{ matrix.module }})
Expand All @@ -252,8 +277,8 @@ jobs:
platforms: ${{ steps.params.outputs.platforms }}
push: true
tags: ${{ steps.params.outputs.image_url }}
cache-from: type=gha,scope=${{ steps.params.outputs.cache_scope }}
cache-to: type=gha,scope=${{ steps.params.outputs.cache_scope }},mode=max
cache-from: ${{ steps.params.outputs.cache_from }}
cache-to: ${{ steps.params.outputs.cache_to_max }}

update_latest_hash:
needs: [prepare, publish]
Expand Down
56 changes: 37 additions & 19 deletions .github/workflows/_publish_pd_store_server_reusable.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,23 @@ jobs:
echo "EOF"
} >> "$GITHUB_OUTPUT"

- name: Summarize build cache strategy
env:
MODE: ${{ inputs.mode }}
run: |
{
echo "## PD / Store / Server image build"
echo
echo "- Cache backend: Docker Hub registry (BuildKit, mode=max)"
echo "- Cache channel: \`$MODE\`"
echo "- Cache writers: integration precheck and amd64 publish jobs"
echo "- Cache reader: arm64 publish jobs"
echo "- PD: \`hugegraph/pd:buildcache-$MODE\`"
echo "- Store: \`hugegraph/store:buildcache-$MODE\`"
echo "- Server (HStore): \`hugegraph/server:buildcache-$MODE\`"
echo "- Server (standalone): \`hugegraph/hugegraph:buildcache-$MODE\`"
} >> "$GITHUB_STEP_SUMMARY"

integration_precheck:
needs: prepare
if: ${{ needs.prepare.outputs.need_update == 'true' && inputs.strict_mode }}
Expand All @@ -164,7 +181,7 @@ jobs:
fi

- name: Checkout source
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
repository: ${{ env.REPOSITORY_URL }}
ref: ${{ env.SOURCE_SHA }}
Expand Down Expand Up @@ -194,8 +211,8 @@ jobs:
platforms: linux/amd64
load: true
tags: hg-ci/pd:precheck
cache-from: type=gha,scope=pd-store-server-pd-amd64
cache-to: type=gha,scope=pd-store-server-pd-amd64,mode=min,ignore-error=true
cache-from: type=registry,ref=hugegraph/pd:buildcache-${{ inputs.mode }}
cache-to: type=registry,ref=hugegraph/pd:buildcache-${{ inputs.mode }},mode=max
build-args: ${{ env.MVN_ARGS }}

- name: Build x86 Store image for integration check
Expand All @@ -206,8 +223,8 @@ jobs:
platforms: linux/amd64
load: true
tags: hg-ci/store:precheck
cache-from: type=gha,scope=pd-store-server-store-amd64
cache-to: type=gha,scope=pd-store-server-store-amd64,mode=min,ignore-error=true
cache-from: type=registry,ref=hugegraph/store:buildcache-${{ inputs.mode }}
cache-to: type=registry,ref=hugegraph/store:buildcache-${{ inputs.mode }},mode=max
build-args: ${{ env.MVN_ARGS }}

- name: Build x86 Server(hstore) image for integration check
Expand All @@ -218,8 +235,8 @@ jobs:
platforms: linux/amd64
load: true
tags: hg-ci/server:precheck
cache-from: type=gha,scope=pd-store-server-server-hstore-amd64
cache-to: type=gha,scope=pd-store-server-server-hstore-amd64,mode=min,ignore-error=true
cache-from: type=registry,ref=hugegraph/server:buildcache-${{ inputs.mode }}
cache-to: type=registry,ref=hugegraph/server:buildcache-${{ inputs.mode }},mode=max
build-args: ${{ env.MVN_ARGS }}

- name: Start compose stack with local images
Expand Down Expand Up @@ -290,22 +307,23 @@ jobs:
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
VERSION_TAG: ${{ needs.prepare.outputs.version_tag }}
MVN_ARGS: ${{ inputs.mvn_args }}
MODE: ${{ inputs.mode }}
steps:
- name: Resolve tags (${{ matrix.module }})
id: tags
env:
IMAGE_REPO: ${{ matrix.image_repo }}
run: |
image_amd64="${IMAGE_REPO}:${VERSION_TAG}-amd64"
module_cache_scope="pd-store-server-${{ matrix.module }}-amd64"
module_cache_ref="${IMAGE_REPO}:buildcache-${MODE}"

{
echo "image_amd64=$image_amd64"
echo "module_cache_scope=$module_cache_scope"
echo "module_cache_ref=$module_cache_ref"
} >> "$GITHUB_OUTPUT"

- name: Checkout source (${{ matrix.module }})
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
repository: ${{ env.REPOSITORY_URL }}
ref: ${{ env.SOURCE_SHA }}
Expand All @@ -331,8 +349,8 @@ jobs:
platforms: linux/amd64
load: true
tags: ${{ steps.tags.outputs.image_amd64 }}
cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }}
cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true
cache-from: type=registry,ref=${{ steps.tags.outputs.module_cache_ref }}
cache-to: type=registry,ref=${{ steps.tags.outputs.module_cache_ref }},mode=max
build-args: ${{ env.MVN_ARGS }}

- name: Build and push amd64 image (${{ matrix.module }})
Expand All @@ -344,8 +362,8 @@ jobs:
platforms: linux/amd64
push: true
tags: ${{ steps.tags.outputs.image_amd64 }}
cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }}
cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true
cache-from: type=registry,ref=${{ steps.tags.outputs.module_cache_ref }}
cache-to: type=registry,ref=${{ steps.tags.outputs.module_cache_ref }},mode=max
build-args: ${{ env.MVN_ARGS }}

- name: Smoke test standalone server amd64
Expand Down Expand Up @@ -386,22 +404,23 @@ jobs:
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
VERSION_TAG: ${{ needs.prepare.outputs.version_tag }}
MVN_ARGS: ${{ inputs.mvn_args }}
MODE: ${{ inputs.mode }}
steps:
- name: Resolve tags (${{ matrix.module }})
id: tags
env:
IMAGE_REPO: ${{ matrix.image_repo }}
run: |
image_arm64="${IMAGE_REPO}:${VERSION_TAG}-arm64"
module_cache_scope="pd-store-server-${{ matrix.module }}-arm64"
module_cache_ref="${IMAGE_REPO}:buildcache-${MODE}"

{
echo "image_arm64=$image_arm64"
echo "module_cache_scope=$module_cache_scope"
echo "module_cache_ref=$module_cache_ref"
} >> "$GITHUB_OUTPUT"

- name: Checkout source (${{ matrix.module }})
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
repository: ${{ env.REPOSITORY_URL }}
ref: ${{ env.SOURCE_SHA }}
Expand Down Expand Up @@ -429,8 +448,7 @@ jobs:
platforms: linux/arm64
push: true
tags: ${{ steps.tags.outputs.image_arm64 }}
cache-from: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }}
cache-to: type=gha,scope=${{ steps.tags.outputs.module_cache_scope }},mode=min,ignore-error=true
cache-from: type=registry,ref=${{ steps.tags.outputs.module_cache_ref }}
build-args: ${{ env.MVN_ARGS }}

publish_manifest:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/publish_latest_hubble_image.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ jobs:
"dockerfile": "./hugegraph-hubble/Dockerfile",
"image_repo_latest": "hugegraph/hubble",
"image_repo_release": "hugegraph/hubble",
"registry_cache_image": "hugegraph/hubble:buildcache-latest",
"platforms_latest": "linux/amd64,linux/arm64",
"platforms_release": "linux/amd64,linux/arm64",
"smoke_test": false
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/publish_release_hubble_image.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
"dockerfile": "./hugegraph-hubble/Dockerfile",
"image_repo_latest": "hugegraph/hubble",
"image_repo_release": "hugegraph/hubble",
"registry_cache_image": "hugegraph/hubble:buildcache-release",
"platforms_latest": "linux/amd64,linux/arm64",
"platforms_release": "linux/amd64,linux/arm64",
"smoke_test": false
Expand Down