From 7ca49447eda359448ec909ca5b04ed820d5faa88 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 15:44:20 +0000 Subject: [PATCH 1/6] Make workflow image name overrideable from Actions vars Keep nikolaik/python-nodejs as the default image name in the workflow, but resolve it through a GitHub Actions variable so forks can publish to a different image without changing the repository. Forks can set IMAGE_NAME in Actions variables or in workflow run configuration. That keeps the default behavior unchanged here while avoiding fork-specific edits in PRs. --- .github/workflows/build.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1a548e..817c001 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,6 +6,9 @@ on: schedule: - cron: "0 00,12 * * *" # Twice a day +env: + IMAGE_NAME: ${{ vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + jobs: generate-matrix: name: Generate build matrix @@ -59,12 +62,12 @@ jobs: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile load: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} # Test - name: Run smoke tests run: | - docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" # Push image - name: Push image @@ -75,7 +78,7 @@ jobs: file: dockerfiles/${{ matrix.key }}.Dockerfile platforms: ${{ join(matrix.platforms) }} push: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} # Store build context - name: Add digest to build context From e59a27ece5c8fa56bc2039d6ca195662b7b1de57 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 17:27:07 +0000 Subject: [PATCH 2/6] Add manual deploy trigger with run-time overrides Allow manual workflow runs from GitHub Actions with a force option and an optional IMAGE_NAME override. This makes it possible to test publish flows without overwriting the real published images tags. --- .github/workflows/build.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 817c001..2e5ad62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,9 +5,21 @@ on: branches: [main] schedule: - cron: "0 00,12 * * *" # Twice a day + workflow_dispatch: + inputs: + force: + description: Force rebuild and republish all image tags + required: false + default: true + type: boolean + image_name: + description: Override image name for this manual run + required: false + default: "" + type: string env: - IMAGE_NAME: ${{ vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + IMAGE_NAME: ${{ inputs.image_name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} jobs: generate-matrix: @@ -26,7 +38,7 @@ jobs: - name: Generate build matrix id: set-matrix run: | - FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) + FORCE=$(if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then echo "--force"; elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) uv run dpn $FORCE build-matrix --event ${{ github.event_name }} From 95ce7111e16d26698a7556b3c88588a42729c9a7 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 21:29:04 +0100 Subject: [PATCH 3/6] Build images natively per architecture Replace the QEMU-based multi-platform build with native amd64 and arm64 runner jobs, then publish the final image tags by assembling a manifest from the architecture-specific tags. Fixes #258. Run the smoke test suite against each architecture-specific image before publishing the final manifest, instead of only testing the locally loaded amd64 image. The build-matrix helper now emits an architecture-expanded matrix for the workflow, and the new unit test covers that expansion. Fixes #314. --- .github/workflows/build.yaml | 65 +++++++++++++++--------- src/docker_python_nodejs/build_matrix.py | 33 ++++++++++-- tests/test_all.py | 38 ++++++++++++++ 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e5ad62..a9090fb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,7 +27,8 @@ jobs: runs-on: ubuntu-latest needs: [test] outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + version_matrix: ${{ steps.set-matrix.outputs.matrix }} + arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -42,13 +43,14 @@ jobs: uv run dpn $FORCE build-matrix --event ${{ github.event_name }} - deploy: - name: ${{ matrix.key }} - runs-on: ubuntu-latest - if: needs.generate-matrix.outputs.matrix != '' + build-arch: + name: ${{ matrix.key }} (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + if: needs.generate-matrix.outputs.arch_matrix != '' needs: [generate-matrix] strategy: - matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }} steps: # Setup - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -56,9 +58,9 @@ jobs: with: enable-cache: true - name: Generate Dockerfile from config - run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}' - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + run: | + context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')" + uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Login to Docker Hub @@ -67,36 +69,51 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # Build + # Build and push - name: Build image + id: build-and-push uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile - load: true - tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} + platforms: ${{ matrix.platform }} + push: true + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" - # Push image - - name: Push image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - id: build-and-push + deploy: + name: Publish ${{ matrix.key }} + runs-on: ubuntu-latest + if: needs.generate-matrix.outputs.version_matrix != '' + needs: [generate-matrix, build-arch] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: - context: . - file: dockerfiles/${{ matrix.key }}.Dockerfile - platforms: ${{ join(matrix.platforms) }} - push: true - tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Publish multi-arch manifest + run: | + tags=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + tags+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker buildx imagetools create --tag "${IMAGE_NAME}:${{ matrix.key }}" "${tags[@]}" - # Store build context - name: Add digest to build context run: | mkdir builds/ - digest="${{ steps.build-and-push.outputs.digest }}" + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" - name: Upload build context diff --git a/src/docker_python_nodejs/build_matrix.py b/src/docker_python_nodejs/build_matrix.py index 6d8fd43..80e7e85 100644 --- a/src/docker_python_nodejs/build_matrix.py +++ b/src/docker_python_nodejs/build_matrix.py @@ -25,7 +25,32 @@ def _github_action_set_output(key: str, value: str) -> None: sys.exit(1) with Path(GITHUB_OUTPUT).open("a") as fp: - fp.write(f"{key}={value}") + fp.write(f"{key}={value}\n") + + +def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str: + return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" + + +def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str: + if not new_or_updated: + return "" + + include: list[dict[str, object]] = [] + for version in new_or_updated: + include.extend( + ( + dataclasses.asdict(version) + | { + "platform": platform, + "arch": platform.split("/")[1], + "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest", + } + ) + for platform in version.platforms + ) + + return json.dumps({"include": include}) def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: @@ -33,7 +58,9 @@ def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.") return - matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" - _github_action_set_output("MATRIX", matrix) + matrix = _build_matrix_json(new_or_updated) + arch_matrix = _build_arch_matrix_json(new_or_updated) + _github_action_set_output("matrix", matrix) + _github_action_set_output("arch_matrix", arch_matrix) logger.info("\n# New or updated versions:") logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated)) diff --git a/tests/test_all.py b/tests/test_all.py index 433e113..d2390ae 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -8,6 +8,7 @@ import pytest import responses +from docker_python_nodejs.build_matrix import _build_arch_matrix_json from docker_python_nodejs.dockerfiles import render_dockerfile_with_context from docker_python_nodejs.readme import update_dynamic_readme from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH @@ -289,3 +290,40 @@ def test_find_new_or_updated_with_digest() -> None: res = find_new_or_updated([new], {existing.key: existing}) assert len(res) == 0 + + +def test_build_arch_matrix_json(build_version: BuildVersion) -> None: + matrix = json.loads(_build_arch_matrix_json([build_version])) + + assert matrix == { + "include": [ + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/amd64", + "arch": "amd64", + "runner": "ubuntu-latest", + }, + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/arm64", + "arch": "arm64", + "runner": "ubuntu-24.04-arm", + }, + ], + } From 50e6cee657fa2d41ca99888e73982900902fda4e Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 21:10:41 +0000 Subject: [PATCH 4/6] Build and test architecture images before push Split the per-architecture workflow so each image is built locally, smoke-tested, and only then pushed to Docker Hub as an architecture-specific tag. This avoids publishing untested architecture images and keeps Docker Hub out of the build phase, so public base-image pulls are no longer attributed to the authenticated Docker Hub account. --- .github/workflows/build.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a9090fb..f7d7167 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -63,27 +63,30 @@ jobs: uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - # Build and push + # Build - name: Build image - id: build-and-push uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile platforms: ${{ matrix.platform }} - push: true + load: true tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm ${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + + # Push + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push image + run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}" deploy: name: Publish ${{ matrix.key }} From 6f07c016beb39791a6e5a08fb3f28544671758ec Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 22:51:32 +0100 Subject: [PATCH 5/6] Create manifests locally before pushing to Docker Hub Replace the deploy-side use of `docker buildx imagetools create` with a local `docker manifest create` followed by authenticated `docker manifest push`. This keeps the manifest assembly step anonymous and delays Docker Hub login until the actual publish step, which reduces authenticated rate-limit failures during multi-arch manifest publication. --- .github/workflows/build.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f7d7167..1577681 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -97,21 +97,25 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Create local multi-arch manifest + run: | + refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}" + - name: Login to Docker Hub uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Publish multi-arch manifest - run: | - tags=("${IMAGE_NAME}:${{ matrix.key }}-amd64") - if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then - tags+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") - fi - docker buildx imagetools create --tag "${IMAGE_NAME}:${{ matrix.key }}" "${tags[@]}" + - name: Push multi-arch manifest + run: docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Add digest to build context run: | From aacaa2128a1efe4cdf28fd451bc0c191601d220f Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 23:06:44 +0100 Subject: [PATCH 6/6] Sign published image manifests with cosign Add keyless cosign signing for the published multi-arch manifests, including the canonical version tags and latest. The workflow now requests the OIDC token permission needed for GitHub-backed signing and signs the final manifest digests after publication. --- .github/workflows/build.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1577681..0abf017 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -21,6 +21,10 @@ on: env: IMAGE_NAME: ${{ inputs.image_name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} +permissions: + contents: write + id-token: write + jobs: generate-matrix: name: Generate build matrix @@ -112,15 +116,21 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push multi-arch manifest - run: docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + id: push-manifest + run: | + digest="$(docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" | tail -n1)" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Install Cosign + uses: sigstore/cosign-installer@v4.0.0 + + - name: Sign multi-arch manifest + run: cosign sign --yes "${IMAGE_NAME}@${{ steps.push-manifest.outputs.digest }}" - name: Add digest to build context run: | mkdir builds/ - digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')" + digest="${{ steps.push-manifest.outputs.digest }}" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" - name: Upload build context