Make latest a multi-arch tag in the normal deploy flow

.github/workflows/build.yaml

@@ -5,14 +5,29 @@ on:
     branches: [main]
   schedule:
     - cron: "0 00,12 * * *"  # Twice a day
+  workflow_dispatch:
+    inputs:
+      force:
+        description: Force rebuild and republish all image tags
+        default: true
+        type: boolean
+      image-name:
+        description: Override image name for this manual run
+        default: ""
+        type: string
+
+env:
+  IMAGE_NAME: ${{ inputs.image-name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }}
 
 jobs:
   generate-matrix:
     name: Generate build matrix
     runs-on: ubuntu-latest
     needs: [test]
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      version_matrix: ${{ steps.set-matrix.outputs.matrix }}
+      arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }}
+      latest_key: ${{ steps.set-matrix.outputs.latest_key }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
@@ -23,66 +38,114 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi)
+          FORCE=
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then
+            FORCE="--force"
+          elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then
+            FORCE="--force"
+          fi
           uv run dpn $FORCE build-matrix --event ${{ github.event_name }}
 
 
-  deploy:
-    name: ${{ matrix.key }}
-    runs-on: ubuntu-latest
-    if: needs.generate-matrix.outputs.matrix != ''
+  build-arch:
+    name: ${{ matrix.key }} (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    if: needs.generate-matrix.outputs.arch_matrix != ''
     needs: [generate-matrix]
     strategy:
-      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }}
     steps:
       # Setup
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
       - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
         with:
           enable-cache: true
       - name: Generate Dockerfile from config
-        run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}'
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4
+        run: |
+          context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')"
+          uv run dpn dockerfile --context "${context}"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
-      - name: Login to Docker Hub
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       # Build
       - name: Build image
         uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
         with:
           context: .
           file: dockerfiles/${{ matrix.key }}.Dockerfile
+          platforms: ${{ matrix.platform }}
           load: true
-          tags: nikolaik/python-nodejs:${{ matrix.key }}
+          tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }}
 
       # Test
       - name: Run smoke tests
         run: |
-          docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version"
+          docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c \
+            "node --version && npm --version && yarn --version && \
+            python --version && pip --version && pipenv --version && \
+            poetry --version && uv --version"
 
-      # Push image
+      # Push
+      - name: Login to Docker Hub
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Push image
-        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7
-        id: build-and-push
+        run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}"
+
+  deploy:
+    name: Publish ${{ matrix.key }}
+    runs-on: ubuntu-latest
+    if: needs.generate-matrix.outputs.version_matrix != ''
+    needs: [generate-matrix, build-arch]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }}
+    steps:
+      - name: Create local multi-arch manifest
+        run: |
+          refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64")
+          if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then
+            refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64")
+          fi
+          docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}"
+          if [[ "${{ needs.generate-matrix.outputs.latest_key }}" == "${{ matrix.key }}" ]]; then
+            docker manifest create "${IMAGE_NAME}:latest" "${refs[@]}"
+          fi
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4
         with:
-          context: .
-          file: dockerfiles/${{ matrix.key }}.Dockerfile
-          platforms: ${{ join(matrix.platforms) }}
-          push: true
-          tags: nikolaik/python-nodejs:${{ matrix.key }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Install regctl
+        uses: regclient/actions/regctl-installer@1b705e32d40851370799ea5814e83d0a5f6a70dc # v0.1.0
+
+      - name: Push multi-arch manifest
+        id: push-manifest
+        run: |
+          docker manifest push "${IMAGE_NAME}:${{ matrix.key }}"
+          digest="$(regctl image digest "${IMAGE_NAME}:${{ matrix.key }}")"
+          echo "digest=${digest}" >> "$GITHUB_OUTPUT"
+
+      - name: Push latest manifest
+        if: needs.generate-matrix.outputs.latest_key == matrix.key
+        run: docker manifest push "${IMAGE_NAME}:latest"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4
 
-      # Store build context
       - name: Add digest to build context
         run: |
           mkdir builds/
-          digest="${{ steps.build-and-push.outputs.digest }}"
-          echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json"
+          digest="${{ steps.push-manifest.outputs.digest }}"
+          echo '${{ toJSON(matrix) }}' |
+            jq --arg digest "$digest" \
+              '. +={"digest": $digest}' \
+              >> "builds/${{ matrix.key }}.json"
 
       - name: Upload build context
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0

README.md

@@ -139,8 +139,6 @@ Versions are kept up to date using official sources. For Python we scrape the _S
 ```bash
 # Pull from Docker Hub
 docker pull nikolaik/python-nodejs:latest
-# Build from GitHub
-docker build -t nikolaik/python-nodejs github.com/nikolaik/docker-python-nodejs
 # Run image
 docker run -it nikolaik/python-nodejs bash
 ```

src/docker_python_nodejs/build_matrix.py

@@ -9,6 +9,8 @@
 if TYPE_CHECKING:
     from .versions import BuildVersion
 
+from .versions import latest_tag_key
+
 CI_EVENT_SCHEDULED = "scheduled"
 
 logger = logging.getLogger("dpn")
@@ -25,15 +27,44 @@ def _github_action_set_output(key: str, value: str) -> None:
         sys.exit(1)
 
     with Path(GITHUB_OUTPUT).open("a") as fp:
-        fp.write(f"{key}={value}")
+        fp.write(f"{key}={value}\n")
+
+
+def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
+
+
+def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str:
+    if not new_or_updated:
+        return ""
+
+    include: list[dict[str, object]] = []
+    for version in new_or_updated:
+        include.extend(
+            (
+                dataclasses.asdict(version)
+                | {
+                    "platform": platform,
+                    "arch": platform.split("/")[1],
+                    "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest",
+                }
+            )
+            for platform in version.platforms
+        )
+
+    return json.dumps({"include": include})
 
 
 def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None:
     if not new_or_updated and ci_event == CI_EVENT_SCHEDULED:
         logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.")
         return
 
-    matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else ""
-    _github_action_set_output("MATRIX", matrix)
+    matrix = _build_matrix_json(new_or_updated)
+    arch_matrix = _build_arch_matrix_json(new_or_updated)
+    latest_key = latest_tag_key(new_or_updated) if new_or_updated else ""
+    _github_action_set_output("matrix", matrix)
+    _github_action_set_output("arch_matrix", arch_matrix)
+    _github_action_set_output("latest_key", latest_key)
     logger.info("\n# New or updated versions:")
     logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated))

src/docker_python_nodejs/nodejs_versions.py

@@ -1,4 +1,5 @@
 import datetime
+import re
 from typing import TypedDict
 
 import requests
@@ -29,6 +30,17 @@ def fetch_node_unofficial_releases() -> list[NodeRelease]:
     return data
 
 
+def fetch_latest_nodejs_version() -> str:
+    url = "https://nodejs.org/dist/latest/SHASUMS256.txt"
+    res = requests.get(url, timeout=10.0)
+    res.raise_for_status()
+    match = re.search(r"node-(v\d+\.\d+\.\d+)-", res.text)
+    if not match:
+        msg = "Could not determine latest Node.js version from SHASUMS256.txt"
+        raise ValueError(msg)
+    return match.group(1)
+
+
 class ReleaseScheduleItem(TypedDict):
     start: str
     lts: str

src/docker_python_nodejs/versions.py

@@ -14,6 +14,7 @@
 
 from .docker_hub import DockerImageDict, DockerTagDict, fetch_tags
 from .nodejs_versions import (
+    fetch_latest_nodejs_version,
     fetch_node_releases,
     fetch_node_unofficial_releases,
     fetch_nodejs_release_schedule,
@@ -105,6 +106,17 @@ def _latest_patch(tags: list[DockerTagDict], ver: str, distro: str) -> str | Non
     return sorted(tags, key=lambda x: Version.parse(x["name"]), reverse=True)[0]["name"] if tags else None
 
 
+def _latest_python_minor(distro: str) -> str:
+    python_patch_re = re.compile(rf"^(\d+\.\d+\.\d+)-{distro}$")
+    tags = [tag["name"] for tag in fetch_tags("python") if python_patch_re.match(tag["name"])]
+    if not tags:
+        msg = f"Could not determine latest Python version for distro '{distro}'"
+        raise ValueError(msg)
+
+    latest_patch = sorted(tags, key=lambda x: Version.parse(x.removesuffix(f"-{distro}")), reverse=True)[0]
+    return ".".join(latest_patch.removesuffix(f"-{distro}").split(".")[:2])
+
+
 def scrape_supported_python_versions() -> list[SupportedVersion]:
     """Scrape supported python versions (risky)."""
     versions = []
@@ -256,6 +268,18 @@ def decide_version_combinations(
     return version_combinations(nodejs_versions, python_versions)
 
 
+def latest_tag_key(versions: list[BuildVersion]) -> str:
+    python_minor = _latest_python_minor(DEFAULT_DISTRO)
+    node_major = fetch_latest_nodejs_version().removeprefix("v").split(".")[0]
+    key = f"python{python_minor}-nodejs{node_major}"
+
+    if key not in {version.key for version in versions}:
+        msg = f"Computed latest tag '{key}' was not part of the current build set"
+        raise ValueError(msg)
+
+    return key
+
+
 def persist_versions(versions: list[BuildVersion], dry_run: bool = False) -> None:
     if dry_run:
         logger.debug(versions)