diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1a548e..f56568f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,19 @@ on: branches: [main] schedule: - cron: "0 00,12 * * *" # Twice a day + workflow_dispatch: + inputs: + force: + description: Force rebuild and republish all image tags + default: true + type: boolean + image-name: + description: Override image name for this manual run + default: "" + type: string + +env: + IMAGE_NAME: ${{ inputs.image-name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} jobs: generate-matrix: @@ -12,7 +25,9 @@ jobs: runs-on: ubuntu-latest needs: [test] outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + version_matrix: ${{ steps.set-matrix.outputs.matrix }} + arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }} + latest_key: ${{ steps.set-matrix.outputs.latest_key }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -23,17 +38,23 @@ jobs: - name: Generate build matrix id: set-matrix run: | - FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) + FORCE= + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then + FORCE="--force" + elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then + FORCE="--force" + fi uv run dpn $FORCE build-matrix --event ${{ github.event_name }} - deploy: - name: ${{ matrix.key }} - runs-on: ubuntu-latest - if: needs.generate-matrix.outputs.matrix != '' + build-arch: + name: ${{ matrix.key }} (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + if: needs.generate-matrix.outputs.arch_matrix != '' needs: [generate-matrix] strategy: - matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }} steps: # Setup - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -41,16 +62,11 @@ jobs: with: enable-cache: true - name: Generate Dockerfile from config - run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}' - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + run: | + context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')" + uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} # Build - name: Build image @@ -58,31 +74,78 @@ jobs: with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile + platforms: ${{ matrix.platform }} load: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c \ + "node --version && npm --version && yarn --version && \ + python --version && pip --version && pipenv --version && \ + poetry --version && uv --version" - # Push image + # Push + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - id: build-and-push + run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}" + + deploy: + name: Publish ${{ matrix.key }} + runs-on: ubuntu-latest + if: needs.generate-matrix.outputs.version_matrix != '' + needs: [generate-matrix, build-arch] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} + steps: + - name: Create local multi-arch manifest + run: | + refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}" + if [[ "${{ needs.generate-matrix.outputs.latest_key }}" == "${{ matrix.key }}" ]]; then + docker manifest create "${IMAGE_NAME}:latest" "${refs[@]}" + fi + + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: - context: . - file: dockerfiles/${{ matrix.key }}.Dockerfile - platforms: ${{ join(matrix.platforms) }} - push: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Install regctl + uses: regclient/actions/regctl-installer@1b705e32d40851370799ea5814e83d0a5f6a70dc # v0.1.0 + + - name: Push multi-arch manifest + id: push-manifest + run: | + docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" + digest="$(regctl image digest "${IMAGE_NAME}:${{ matrix.key }}")" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + + - name: Push latest manifest + if: needs.generate-matrix.outputs.latest_key == matrix.key + run: docker manifest push "${IMAGE_NAME}:latest" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - # Store build context - name: Add digest to build context run: | mkdir builds/ - digest="${{ steps.build-and-push.outputs.digest }}" - echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" + digest="${{ steps.push-manifest.outputs.digest }}" + echo '${{ toJSON(matrix) }}' | + jq --arg digest "$digest" \ + '. +={"digest": $digest}' \ + >> "builds/${{ matrix.key }}.json" - name: Upload build context uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 10937f1..0000000 --- a/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:trixie -LABEL org.opencontainers.image.authors="Nikolai R Kristiansen " - -SHELL ["/bin/bash", "-euo", "pipefail", "-c"] -RUN groupadd --gid 1000 pn && useradd --uid 1000 --gid pn --shell /bin/bash --create-home pn -ENV POETRY_HOME=/usr/local - -RUN NODE_VERSION="$(curl -fsSL https://nodejs.org/dist/latest/SHASUMS256.txt | head -n1 | awk '{ print $2}' | awk -F - '{ print $2}')" \ - ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - arm64) ARCH='arm64';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && for key in $(curl -sL https://raw.githubusercontent.com/nodejs/docker-node/HEAD/keys/node.keys); do \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs -RUN npm install -g corepack && corepack enable yarn -RUN \ - apt-get update && \ - apt-get upgrade -yqq && \ - pip install -U pip pipenv uv && \ - curl -sSL https://install.python-poetry.org | python - && \ - rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index ee7bbd0..85f8fb8 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,6 @@ Versions are kept up to date using official sources. For Python we scrape the _S ```bash # Pull from Docker Hub docker pull nikolaik/python-nodejs:latest -# Build from GitHub -docker build -t nikolaik/python-nodejs github.com/nikolaik/docker-python-nodejs # Run image docker run -it nikolaik/python-nodejs bash ``` diff --git a/src/docker_python_nodejs/build_matrix.py b/src/docker_python_nodejs/build_matrix.py index 6d8fd43..6deee3f 100644 --- a/src/docker_python_nodejs/build_matrix.py +++ b/src/docker_python_nodejs/build_matrix.py @@ -9,6 +9,8 @@ if TYPE_CHECKING: from .versions import BuildVersion +from .versions import latest_tag_key + CI_EVENT_SCHEDULED = "scheduled" logger = logging.getLogger("dpn") @@ -25,7 +27,32 @@ def _github_action_set_output(key: str, value: str) -> None: sys.exit(1) with Path(GITHUB_OUTPUT).open("a") as fp: - fp.write(f"{key}={value}") + fp.write(f"{key}={value}\n") + + +def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str: + return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" + + +def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str: + if not new_or_updated: + return "" + + include: list[dict[str, object]] = [] + for version in new_or_updated: + include.extend( + ( + dataclasses.asdict(version) + | { + "platform": platform, + "arch": platform.split("/")[1], + "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest", + } + ) + for platform in version.platforms + ) + + return json.dumps({"include": include}) def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: @@ -33,7 +60,11 @@ def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.") return - matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" - _github_action_set_output("MATRIX", matrix) + matrix = _build_matrix_json(new_or_updated) + arch_matrix = _build_arch_matrix_json(new_or_updated) + latest_key = latest_tag_key(new_or_updated) if new_or_updated else "" + _github_action_set_output("matrix", matrix) + _github_action_set_output("arch_matrix", arch_matrix) + _github_action_set_output("latest_key", latest_key) logger.info("\n# New or updated versions:") logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated)) diff --git a/src/docker_python_nodejs/nodejs_versions.py b/src/docker_python_nodejs/nodejs_versions.py index 3ce5db4..ac5c57b 100644 --- a/src/docker_python_nodejs/nodejs_versions.py +++ b/src/docker_python_nodejs/nodejs_versions.py @@ -1,4 +1,5 @@ import datetime +import re from typing import TypedDict import requests @@ -29,6 +30,17 @@ def fetch_node_unofficial_releases() -> list[NodeRelease]: return data +def fetch_latest_nodejs_version() -> str: + url = "https://nodejs.org/dist/latest/SHASUMS256.txt" + res = requests.get(url, timeout=10.0) + res.raise_for_status() + match = re.search(r"node-(v\d+\.\d+\.\d+)-", res.text) + if not match: + msg = "Could not determine latest Node.js version from SHASUMS256.txt" + raise ValueError(msg) + return match.group(1) + + class ReleaseScheduleItem(TypedDict): start: str lts: str diff --git a/src/docker_python_nodejs/versions.py b/src/docker_python_nodejs/versions.py index 8b22f4f..41ca6bc 100644 --- a/src/docker_python_nodejs/versions.py +++ b/src/docker_python_nodejs/versions.py @@ -14,6 +14,7 @@ from .docker_hub import DockerImageDict, DockerTagDict, fetch_tags from .nodejs_versions import ( + fetch_latest_nodejs_version, fetch_node_releases, fetch_node_unofficial_releases, fetch_nodejs_release_schedule, @@ -105,6 +106,17 @@ def _latest_patch(tags: list[DockerTagDict], ver: str, distro: str) -> str | Non return sorted(tags, key=lambda x: Version.parse(x["name"]), reverse=True)[0]["name"] if tags else None +def _latest_python_minor(distro: str) -> str: + python_patch_re = re.compile(rf"^(\d+\.\d+\.\d+)-{distro}$") + tags = [tag["name"] for tag in fetch_tags("python") if python_patch_re.match(tag["name"])] + if not tags: + msg = f"Could not determine latest Python version for distro '{distro}'" + raise ValueError(msg) + + latest_patch = sorted(tags, key=lambda x: Version.parse(x.removesuffix(f"-{distro}")), reverse=True)[0] + return ".".join(latest_patch.removesuffix(f"-{distro}").split(".")[:2]) + + def scrape_supported_python_versions() -> list[SupportedVersion]: """Scrape supported python versions (risky).""" versions = [] @@ -256,6 +268,18 @@ def decide_version_combinations( return version_combinations(nodejs_versions, python_versions) +def latest_tag_key(versions: list[BuildVersion]) -> str: + python_minor = _latest_python_minor(DEFAULT_DISTRO) + node_major = fetch_latest_nodejs_version().removeprefix("v").split(".")[0] + key = f"python{python_minor}-nodejs{node_major}" + + if key not in {version.key for version in versions}: + msg = f"Computed latest tag '{key}' was not part of the current build set" + raise ValueError(msg) + + return key + + def persist_versions(versions: list[BuildVersion], dry_run: bool = False) -> None: if dry_run: logger.debug(versions) diff --git a/tests/test_all.py b/tests/test_all.py index 433e113..b3b1251 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -8,6 +8,7 @@ import pytest import responses +from docker_python_nodejs.build_matrix import _build_arch_matrix_json, _build_matrix_json from docker_python_nodejs.dockerfiles import render_dockerfile_with_context from docker_python_nodejs.readme import update_dynamic_readme from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH @@ -18,6 +19,7 @@ decide_version_combinations, fetch_supported_nodejs_versions, find_new_or_updated, + latest_tag_key, load_build_contexts, scrape_supported_python_versions, ) @@ -289,3 +291,150 @@ def test_find_new_or_updated_with_digest() -> None: res = find_new_or_updated([new], {existing.key: existing}) assert len(res) == 0 + + +def test_build_arch_matrix_json(build_version: BuildVersion) -> None: + matrix = json.loads(_build_arch_matrix_json([build_version])) + + assert matrix == { + "include": [ + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/amd64", + "arch": "amd64", + "runner": "ubuntu-latest", + }, + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/arm64", + "arch": "arm64", + "runner": "ubuntu-24.04-arm", + }, + ], + } + + +def test_build_matrix_json() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs25", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="25", + nodejs_canonical="25.8.1", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + BuildVersion( + key="python3.14-nodejs24-bookworm", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-bookworm", + nodejs="24", + nodejs_canonical="24.14.0", + distro="bookworm", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + matrix = json.loads(_build_matrix_json(versions)) + + assert matrix == { + "include": [ + { + "key": "python3.14-nodejs25", + "python": "3.14", + "python_canonical": "3.14.3", + "python_image": "3.14.3-trixie", + "nodejs": "25", + "nodejs_canonical": "25.8.1", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + }, + { + "key": "python3.14-nodejs24-bookworm", + "python": "3.14", + "python_canonical": "3.14.3", + "python_image": "3.14.3-bookworm", + "nodejs": "24", + "nodejs_canonical": "24.14.0", + "distro": "bookworm", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + }, + ], + } + + +def test_latest_tag_key_matches_latest_sources() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs25", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="25", + nodejs_canonical="25.8.1", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + BuildVersion( + key="python3.14-nodejs24-bookworm", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-bookworm", + nodejs="24", + nodejs_canonical="24.14.0", + distro="bookworm", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + with ( + mock.patch("docker_python_nodejs.versions._latest_python_minor", return_value="3.14"), + mock.patch("docker_python_nodejs.versions.fetch_latest_nodejs_version", return_value="v25.8.1"), + ): + assert latest_tag_key(versions) == "python3.14-nodejs25" + + +def test_latest_tag_key_fails_if_canonical_build_is_missing() -> None: + versions = [ + BuildVersion( + key="python3.14-nodejs24", + python="3.14", + python_canonical="3.14.3", + python_image="3.14.3-trixie", + nodejs="24", + nodejs_canonical="24.14.0", + distro="trixie", + platforms=["linux/amd64", "linux/arm64"], + ), + ] + + with ( + mock.patch("docker_python_nodejs.versions._latest_python_minor", return_value="3.14"), + mock.patch("docker_python_nodejs.versions.fetch_latest_nodejs_version", return_value="v25.8.1"), + pytest.raises( + ValueError, + match=r"Computed latest tag 'python3\.14-nodejs25' was not part of the current build set", + ), + ): + latest_tag_key(versions)