From 7ca49447eda359448ec909ca5b04ed820d5faa88 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 15:44:20 +0000 Subject: [PATCH 1/9] Make workflow image name overrideable from Actions vars Keep nikolaik/python-nodejs as the default image name in the workflow, but resolve it through a GitHub Actions variable so forks can publish to a different image without changing the repository. Forks can set IMAGE_NAME in Actions variables or in workflow run configuration. That keeps the default behavior unchanged here while avoiding fork-specific edits in PRs. --- .github/workflows/build.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1a548e..817c001 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,6 +6,9 @@ on: schedule: - cron: "0 00,12 * * *" # Twice a day +env: + IMAGE_NAME: ${{ vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + jobs: generate-matrix: name: Generate build matrix @@ -59,12 +62,12 @@ jobs: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile load: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} # Test - name: Run smoke tests run: | - docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" # Push image - name: Push image @@ -75,7 +78,7 @@ jobs: file: dockerfiles/${{ matrix.key }}.Dockerfile platforms: ${{ join(matrix.platforms) }} push: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} # Store build context - name: Add digest to build context From e59a27ece5c8fa56bc2039d6ca195662b7b1de57 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 17:27:07 +0000 Subject: [PATCH 2/9] Add manual deploy trigger with run-time overrides Allow manual workflow runs from GitHub Actions with a force option and an optional IMAGE_NAME override. This makes it possible to test publish flows without overwriting the real published images tags. --- .github/workflows/build.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 817c001..2e5ad62 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,9 +5,21 @@ on: branches: [main] schedule: - cron: "0 00,12 * * *" # Twice a day + workflow_dispatch: + inputs: + force: + description: Force rebuild and republish all image tags + required: false + default: true + type: boolean + image_name: + description: Override image name for this manual run + required: false + default: "" + type: string env: - IMAGE_NAME: ${{ vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + IMAGE_NAME: ${{ inputs.image_name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} jobs: generate-matrix: @@ -26,7 +38,7 @@ jobs: - name: Generate build matrix id: set-matrix run: | - FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) + FORCE=$(if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then echo "--force"; elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) uv run dpn $FORCE build-matrix --event ${{ github.event_name }} From e298eba2cefdfa959135dc7744122597b3c0f6c3 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 21:29:04 +0100 Subject: [PATCH 3/9] Build images natively per architecture Replace the QEMU-based multi-platform build with native amd64 and arm64 runner jobs, then publish the final image tags by assembling a manifest from the architecture-specific tags. Fixes #258. Run the smoke test suite against each architecture-specific image before publishing the final manifest, instead of only testing the locally loaded amd64 image. The build-matrix helper now emits an architecture-expanded matrix for the workflow, and the new unit test covers that expansion. Fixes #314. --- .github/workflows/build.yaml | 65 +++++++++++++++--------- src/docker_python_nodejs/build_matrix.py | 33 ++++++++++-- tests/test_all.py | 38 ++++++++++++++ 3 files changed, 109 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e5ad62..a9090fb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,7 +27,8 @@ jobs: runs-on: ubuntu-latest needs: [test] outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + version_matrix: ${{ steps.set-matrix.outputs.matrix }} + arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -42,13 +43,14 @@ jobs: uv run dpn $FORCE build-matrix --event ${{ github.event_name }} - deploy: - name: ${{ matrix.key }} - runs-on: ubuntu-latest - if: needs.generate-matrix.outputs.matrix != '' + build-arch: + name: ${{ matrix.key }} (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + if: needs.generate-matrix.outputs.arch_matrix != '' needs: [generate-matrix] strategy: - matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }} steps: # Setup - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -56,9 +58,9 @@ jobs: with: enable-cache: true - name: Generate Dockerfile from config - run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}' - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + run: | + context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')" + uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Login to Docker Hub @@ -67,36 +69,51 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # Build + # Build and push - name: Build image + id: build-and-push uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile - load: true - tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} + platforms: ${{ matrix.platform }} + push: true + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" - # Push image - - name: Push image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - id: build-and-push + deploy: + name: Publish ${{ matrix.key }} + runs-on: ubuntu-latest + if: needs.generate-matrix.outputs.version_matrix != '' + needs: [generate-matrix, build-arch] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: - context: . - file: dockerfiles/${{ matrix.key }}.Dockerfile - platforms: ${{ join(matrix.platforms) }} - push: true - tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Publish multi-arch manifest + run: | + tags=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + tags+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker buildx imagetools create --tag "${IMAGE_NAME}:${{ matrix.key }}" "${tags[@]}" - # Store build context - name: Add digest to build context run: | mkdir builds/ - digest="${{ steps.build-and-push.outputs.digest }}" + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" - name: Upload build context diff --git a/src/docker_python_nodejs/build_matrix.py b/src/docker_python_nodejs/build_matrix.py index 6d8fd43..80e7e85 100644 --- a/src/docker_python_nodejs/build_matrix.py +++ b/src/docker_python_nodejs/build_matrix.py @@ -25,7 +25,32 @@ def _github_action_set_output(key: str, value: str) -> None: sys.exit(1) with Path(GITHUB_OUTPUT).open("a") as fp: - fp.write(f"{key}={value}") + fp.write(f"{key}={value}\n") + + +def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str: + return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" + + +def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str: + if not new_or_updated: + return "" + + include: list[dict[str, object]] = [] + for version in new_or_updated: + include.extend( + ( + dataclasses.asdict(version) + | { + "platform": platform, + "arch": platform.split("/")[1], + "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest", + } + ) + for platform in version.platforms + ) + + return json.dumps({"include": include}) def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: @@ -33,7 +58,9 @@ def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.") return - matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" - _github_action_set_output("MATRIX", matrix) + matrix = _build_matrix_json(new_or_updated) + arch_matrix = _build_arch_matrix_json(new_or_updated) + _github_action_set_output("matrix", matrix) + _github_action_set_output("arch_matrix", arch_matrix) logger.info("\n# New or updated versions:") logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated)) diff --git a/tests/test_all.py b/tests/test_all.py index 433e113..d2390ae 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -8,6 +8,7 @@ import pytest import responses +from docker_python_nodejs.build_matrix import _build_arch_matrix_json from docker_python_nodejs.dockerfiles import render_dockerfile_with_context from docker_python_nodejs.readme import update_dynamic_readme from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH @@ -289,3 +290,40 @@ def test_find_new_or_updated_with_digest() -> None: res = find_new_or_updated([new], {existing.key: existing}) assert len(res) == 0 + + +def test_build_arch_matrix_json(build_version: BuildVersion) -> None: + matrix = json.loads(_build_arch_matrix_json([build_version])) + + assert matrix == { + "include": [ + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/amd64", + "arch": "amd64", + "runner": "ubuntu-latest", + }, + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/arm64", + "arch": "arm64", + "runner": "ubuntu-24.04-arm", + }, + ], + } From 127d3b85f4b6b7556ad6bdaccbaccd41a43fb291 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 21:10:41 +0000 Subject: [PATCH 4/9] Build and test architecture images before push Split the per-architecture workflow so each image is built locally, smoke-tested, and only then pushed to Docker Hub as an architecture-specific tag. This avoids publishing untested architecture images and keeps Docker Hub out of the build phase, so public base-image pulls are no longer attributed to the authenticated Docker Hub account. --- .github/workflows/build.yaml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a9090fb..f7d7167 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -63,27 +63,30 @@ jobs: uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - # Build and push + # Build - name: Build image - id: build-and-push uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile platforms: ${{ matrix.platform }} - push: true + load: true tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm ${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + + # Push + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push image + run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}" deploy: name: Publish ${{ matrix.key }} From e247afd417d4791d4a68116b4070e6005299f84e Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Mon, 16 Mar 2026 22:51:32 +0100 Subject: [PATCH 5/9] Create manifests locally before pushing to Docker Hub Replace the deploy-side use of `docker buildx imagetools create` with a local `docker manifest create` followed by authenticated `docker manifest push`. This keeps the manifest assembly step anonymous and delays Docker Hub login until the actual publish step, which reduces authenticated rate-limit failures during multi-arch manifest publication. --- .github/workflows/build.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f7d7167..1577681 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -97,21 +97,25 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Create local multi-arch manifest + run: | + refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}" + - name: Login to Docker Hub uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Publish multi-arch manifest - run: | - tags=("${IMAGE_NAME}:${{ matrix.key }}-amd64") - if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then - tags+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") - fi - docker buildx imagetools create --tag "${IMAGE_NAME}:${{ matrix.key }}" "${tags[@]}" + - name: Push multi-arch manifest + run: docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Add digest to build context run: | From db3ee8d943009e42a0b2765e38bfd14571c1ec74 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Tue, 17 Mar 2026 12:34:31 +0100 Subject: [PATCH 6/9] Make latest a multi-arch tag in the normal deploy flow Determine the canonical latest image from the current build set and publish `latest` during the normal deploy flow alongside the corresponding versioned tag. This replaces the separate `latest` build path, so `latest` is published for all supported architectures without relying on a separate Docker Hub build. Remove the root Dockerfile because it only existed for the separate Docker Hub Automated Build path. Any Docker Hub automated build still configured to use that file must be disabled in Docker Hub. Fixes #263. --- .github/workflows/build.yaml | 15 ++- Dockerfile | 32 ------ README.md | 2 - src/docker_python_nodejs/build_matrix.py | 4 + src/docker_python_nodejs/nodejs_versions.py | 12 +++ src/docker_python_nodejs/versions.py | 24 +++++ tests/test_all.py | 113 +++++++++++++++++++- 7 files changed, 165 insertions(+), 37 deletions(-) delete mode 100644 Dockerfile diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1577681..73ec014 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,6 +29,7 @@ jobs: outputs: version_matrix: ${{ steps.set-matrix.outputs.matrix }} arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }} + latest_key: ${{ steps.set-matrix.outputs.latest_key }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -104,6 +105,9 @@ jobs: refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") fi docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}" + if [[ "${{ needs.generate-matrix.outputs.latest_key }}" == "${{ matrix.key }}" ]]; then + docker manifest create "${IMAGE_NAME}:latest" "${refs[@]}" + fi - name: Login to Docker Hub uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 @@ -112,7 +116,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push multi-arch manifest - run: docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + id: push-manifest + run: | + digest="$(docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" | tail -n1)" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + + - name: Push latest manifest + if: needs.generate-matrix.outputs.latest_key == matrix.key + run: docker manifest push "${IMAGE_NAME}:latest" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 @@ -120,7 +131,7 @@ jobs: - name: Add digest to build context run: | mkdir builds/ - digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')" + digest="${{ steps.push-manifest.outputs.digest }}" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" - name: Upload build context diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 10937f1..0000000 --- a/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:trixie -LABEL org.opencontainers.image.authors="Nikolai R Kristiansen " - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -RUN groupadd --gid 1000 pn && useradd --uid 1000 --gid pn --shell /bin/bash --create-home pn -ENV POETRY_HOME=/usr/local - -RUN NODE_VERSION="$(curl -fsSL https://nodejs.org/dist/latest/SHASUMS256.txt | head -n1 | awk '{ print $2}' | awk -F - '{ print $2}')" \ - ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - arm64) ARCH='arm64';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && for key in $(curl -sL https://raw.githubusercontent.com/nodejs/docker-node/HEAD/keys/node.keys); do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs -RUN npm install -g corepack && corepack enable yarn -RUN \ - apt-get update && \ - apt-get upgrade -yqq && \ - pip install -U pip pipenv uv && \ - curl -sSL https://install.python-poetry.org | python - && \ - rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index ee7bbd0..85f8fb8 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,6 @@ Versions are kept up to date using official sources. For Python we scrape the _S ```bash # Pull from Docker Hub docker pull nikolaik/python-nodejs:latest -# Build from GitHub -docker build -t nikolaik/python-nodejs github.com/nikolaik/docker-python-nodejs # Run image docker run -it nikolaik/python-nodejs bash ``` diff --git a/src/docker_python_nodejs/build_matrix.py b/src/docker_python_nodejs/build_matrix.py index 80e7e85..6deee3f 100644 --- a/src/docker_python_nodejs/build_matrix.py +++ b/src/docker_python_nodejs/build_matrix.py @@ -9,6 +9,8 @@ if TYPE_CHECKING: from .versions import BuildVersion +from .versions import latest_tag_key + CI_EVENT_SCHEDULED = "scheduled" logger = logging.getLogger("dpn") @@ -60,7 +62,9 @@ def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: matrix = _build_matrix_json(new_or_updated) arch_matrix = _build_arch_matrix_json(new_or_updated) + latest_key = latest_tag_key(new_or_updated) if new_or_updated else "" _github_action_set_output("matrix", matrix) _github_action_set_output("arch_matrix", arch_matrix) + _github_action_set_output("latest_key", latest_key) logger.info("\n# New or updated versions:") logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated)) diff --git a/src/docker_python_nodejs/nodejs_versions.py b/src/docker_python_nodejs/nodejs_versions.py index 3ce5db4..ac5c57b 100644 --- a/src/docker_python_nodejs/nodejs_versions.py +++ b/src/docker_python_nodejs/nodejs_versions.py @@ -1,4 +1,5 @@ import datetime +import re from typing import TypedDict import requests @@ -29,6 +30,17 @@ def fetch_node_unofficial_releases() -> list[NodeRelease]: return data +def fetch_latest_nodejs_version() -> str: + url = "https://nodejs.org/dist/latest/SHASUMS256.txt" + res = requests.get(url, timeout=10.0) + res.raise_for_status() + match = re.search(r"node-(v\d+\.\d+\.\d+)-", res.text) + if not match: + msg = "Could not determine latest Node.js version from SHASUMS256.txt" + raise ValueError(msg) + return match.group(1) + + class ReleaseScheduleItem(TypedDict): start: str lts: str diff --git a/src/docker_python_nodejs/versions.py b/src/docker_python_nodejs/versions.py index 8b22f4f..41ca6bc 100644 --- a/src/docker_python_nodejs/versions.py +++ b/src/docker_python_nodejs/versions.py @@ -14,6 +14,7 @@ from .docker_hub import DockerImageDict, DockerTagDict, fetch_tags from .nodejs_versions import ( + fetch_latest_nodejs_version, fetch_node_releases, fetch_node_unofficial_releases, fetch_nodejs_release_schedule, @@ -105,6 +106,17 @@ def _latest_patch(tags: list[DockerTagDict], ver: str, distro: str) -> str | Non return sorted(tags, key=lambda x: Version.parse(x["name"]), reverse=True)[0]["name"] if tags else None +def _latest_python_minor(distro: str) -> str: + python_patch_re = re.compile(rf"^(\d+\.\d+\.\d+)-{distro}$") + tags = [tag["name"] for tag in fetch_tags("python") if python_patch_re.match(tag["name"])] + if not tags: + msg = f"Could not determine latest Python version for distro '{distro}'" + raise ValueError(msg) + + latest_patch = sorted(tags, key=lambda x: Version.parse(x.removesuffix(f"-{distro}")), reverse=True)[0] + return ".".join(latest_patch.removesuffix(f"-{distro}").split(".")[:2]) + + def scrape_supported_python_versions() -> list[SupportedVersion]: """Scrape supported python versions (risky).""" versions = [] @@ -256,6 +268,18 @@ def decide_version_combinations( return version_combinations(nodejs_versions, python_versions) +def latest_tag_key(versions: list[BuildVersion]) -> str: + python_minor = _latest_python_minor(DEFAULT_DISTRO) + node_major = fetch_latest_nodejs_version().removeprefix("v").split(".")[0] + key = f"python{python_minor}-nodejs{node_major}" + + if key not in {version.key for version in versions}: + msg = f"Computed latest tag '{key}' was not part of the current build set" + raise ValueError(msg) + + return key + + def persist_versions(versions: list[BuildVersion], dry_run: bool = False) -> None: if dry_run: logger.debug(versions) diff --git a/tests/test_all.py b/tests/test_all.py index d2390ae..b3b1251 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -8,7 +8,7 @@ import pytest import responses -from docker_python_nodejs.build_matrix import _build_arch_matrix_json +from docker_python_nodejs.build_matrix import _build_arch_matrix_json, _build_matrix_json from docker_python_nodejs.dockerfiles import render_dockerfile_with_context from docker_python_nodejs.readme import update_dynamic_readme from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH @@ -19,6 +19,7 @@ decide_version_combinations, fetch_supported_nodejs_versions, find_new_or_updated, + latest_tag_key, load_build_contexts, scrape_supported_python_versions, ) @@ -327,3 +328,113 @@ def test_build_arch_matrix_json(build_version: BuildVersion) -> None: }, ], } + + +def test_build_matrix_json() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs25", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="25", + nodejs_canonical="25.8.1", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + BuildVersion( + key="python3.14-nodejs24-bookworm", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-bookworm", + nodejs="24", + nodejs_canonical="24.14.0", + distro="bookworm", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + matrix = json.loads(_build_matrix_json(versions)) + + assert matrix == { + "include": [ + { + "key": "python3.14-nodejs25", + "python": "3.14", + "python_canonical": "3.14.3", + "python_image": "3.14.3-trixie", + "nodejs": "25", + "nodejs_canonical": "25.8.1", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + }, + { + "key": "python3.14-nodejs24-bookworm", + "python": "3.14", + "python_canonical": "3.14.3", + "python_image": "3.14.3-bookworm", + "nodejs": "24", + "nodejs_canonical": "24.14.0", + "distro": "bookworm", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + }, + ], + } + + +def test_latest_tag_key_matches_latest_sources() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs25", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="25", + nodejs_canonical="25.8.1", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + BuildVersion( + key="python3.14-nodejs24-bookworm", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-bookworm", + nodejs="24", + nodejs_canonical="24.14.0", + distro="bookworm", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + with ( + mock.patch("docker_python_nodejs.versions._latest_python_minor", return_value="3.14"), + mock.patch("docker_python_nodejs.versions.fetch_latest_nodejs_version", return_value="v25.8.1"), + ): + assert latest_tag_key(versions) == "python3.14-nodejs25" + + +def test_latest_tag_key_fails_if_canonical_build_is_missing() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs24", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="24", + nodejs_canonical="24.14.0", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + with ( + mock.patch("docker_python_nodejs.versions._latest_python_minor", return_value="3.14"), + mock.patch("docker_python_nodejs.versions.fetch_latest_nodejs_version", return_value="v25.8.1"), + pytest.raises( + ValueError, + match=r"Computed latest tag 'python3\.14-nodejs25' was not part of the current build set", + ), + ): + latest_tag_key(versions) From 5dfaea967ed562e6a0bffee6048a7f6d8537462c Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Wed, 18 Mar 2026 11:08:09 +0100 Subject: [PATCH 7/9] Refine build workflow inputs and shell formatting Address review feedback on PR #373 by renaming the manual image override input and reformatting long shell commands for readability. --- .github/workflows/build.yaml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2e5ad62..5ca7f7c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -9,17 +9,15 @@ on: inputs: force: description: Force rebuild and republish all image tags - required: false default: true type: boolean - image_name: + image-name: description: Override image name for this manual run - required: false default: "" type: string env: - IMAGE_NAME: ${{ inputs.image_name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + IMAGE_NAME: ${{ inputs.image-name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} jobs: generate-matrix: @@ -38,7 +36,12 @@ jobs: - name: Generate build matrix id: set-matrix run: | - FORCE=$(if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then echo "--force"; elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) + FORCE= + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then + FORCE="--force" + elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then + FORCE="--force" + fi uv run dpn $FORCE build-matrix --event ${{ github.event_name }} @@ -79,7 +82,10 @@ jobs: # Test - name: Run smoke tests run: | - docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }} sh -c \ + "node --version && npm --version && yarn --version && \ + python --version && pip --version && pipenv --version && \ + poetry --version && uv --version" # Push image - name: Push image @@ -97,7 +103,10 @@ jobs: run: | mkdir builds/ digest="${{ steps.build-and-push.outputs.digest }}" - echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" + echo '${{ toJSON(matrix) }}' | + jq --arg digest "$digest" \ + '. +={"digest": $digest}' \ + >> "builds/${{ matrix.key }}.json" - name: Upload build context uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 From 432021e1207e4f8794b5563dde8756dcb4323225 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Wed, 18 Mar 2026 11:58:57 +0100 Subject: [PATCH 8/9] Use output of manifest-push to get digest of manifest --- .github/workflows/build.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5109a1e..0057a34 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -118,7 +118,10 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push multi-arch manifest - run: docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + id: push-manifest + run: | + digest="$(docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" | tail -n1)" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 @@ -126,7 +129,7 @@ jobs: - name: Add digest to build context run: | mkdir builds/ - digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${{ matrix.key }}" | awk '/^Digest:/ {print $2}')" + digest="${{ steps.push-manifest.outputs.digest }}" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" \ '. +={"digest": $digest}' \ From 92d3ce3f1e0685fdb826123f9a79dbf3dfa39634 Mon Sep 17 00:00:00 2001 From: Stefan Kuhn Date: Fri, 20 Mar 2026 15:33:50 +0000 Subject: [PATCH 9/9] Use regctl to resolve pushed manifest digest --- .github/workflows/build.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0057a34..015c0b7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -117,10 +117,14 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Install regctl + uses: regclient/actions/regctl-installer@1b705e32d40851370799ea5814e83d0a5f6a70dc # v0.1.0 + - name: Push multi-arch manifest id: push-manifest run: | - digest="$(docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" | tail -n1)" + docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + digest="$(regctl image digest "${IMAGE_NAME}:${{ matrix.key }}")" echo "digest=${digest}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx