diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1a548e..0abf017 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,25 @@ on: branches: [main] schedule: - cron: "0 00,12 * * *" # Twice a day + workflow_dispatch: + inputs: + force: + description: Force rebuild and republish all image tags + required: false + default: true + type: boolean + image_name: + description: Override image name for this manual run + required: false + default: "" + type: string + +env: + IMAGE_NAME: ${{ inputs.image_name || vars.IMAGE_NAME || 'nikolaik/python-nodejs' }} + +permissions: + contents: write + id-token: write jobs: generate-matrix: @@ -12,7 +31,8 @@ jobs: runs-on: ubuntu-latest needs: [test] outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + version_matrix: ${{ steps.set-matrix.outputs.matrix }} + arch_matrix: ${{ steps.set-matrix.outputs.arch_matrix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -23,17 +43,18 @@ jobs: - name: Generate build matrix id: set-matrix run: | - FORCE=$(if git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) + FORCE=$(if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force }}" == "true" ]]; then echo "--force"; elif git log --pretty=format:"%s" HEAD^..HEAD | grep -q '\[force\]'; then echo "--force"; else echo ""; fi) uv run dpn $FORCE build-matrix --event ${{ github.event_name }} - deploy: - name: ${{ matrix.key }} - runs-on: ubuntu-latest - if: needs.generate-matrix.outputs.matrix != '' + build-arch: + name: ${{ matrix.key }} (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + if: needs.generate-matrix.outputs.arch_matrix != '' needs: [generate-matrix] strategy: - matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.arch_matrix) }} steps: # Setup - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -41,16 +62,11 @@ jobs: with: enable-cache: true - name: Generate Dockerfile from config - run: uv run dpn dockerfile --context '${{ toJSON(matrix) }}' - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + run: | + context="$(echo '${{ toJSON(matrix) }}' | jq -c '{key, python, python_canonical, python_image, nodejs, nodejs_canonical, distro, platforms, digest}')" + uv run dpn dockerfile --context "${context}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - name: Login to Docker Hub - uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} # Build - name: Build image @@ -58,30 +74,63 @@ jobs: with: context: . file: dockerfiles/${{ matrix.key }}.Dockerfile + platforms: ${{ matrix.platform }} load: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + tags: ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} # Test - name: Run smoke tests run: | - docker run --rm nikolaik/python-nodejs:${{ matrix.key }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" + docker run --rm ${{ env.IMAGE_NAME }}:${{ matrix.key }}-${{ matrix.arch }} sh -c "node --version && npm --version && yarn --version && python --version && pip --version && pipenv --version && poetry --version && uv --version" - # Push image + # Push + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - id: build-and-push + run: docker push "${IMAGE_NAME}:${{ matrix.key }}-${{ matrix.arch }}" + + deploy: + name: Publish ${{ matrix.key }} + runs-on: ubuntu-latest + if: needs.generate-matrix.outputs.version_matrix != '' + needs: [generate-matrix, build-arch] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.version_matrix) }} + steps: + - name: Create local multi-arch manifest + run: | + refs=("${IMAGE_NAME}:${{ matrix.key }}-amd64") + if echo '${{ toJSON(matrix.platforms) }}' | jq -e '.[] == "linux/arm64"' > /dev/null; then + refs+=("${IMAGE_NAME}:${{ matrix.key }}-arm64") + fi + docker manifest create "${IMAGE_NAME}:${{ matrix.key }}" "${refs[@]}" + + - name: Login to Docker Hub + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4 with: - context: . - file: dockerfiles/${{ matrix.key }}.Dockerfile - platforms: ${{ join(matrix.platforms) }} - push: true - tags: nikolaik/python-nodejs:${{ matrix.key }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push multi-arch manifest + id: push-manifest + run: | + digest="$(docker manifest push "${IMAGE_NAME}:${{ matrix.key }}" | tail -n1)" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + + - name: Install Cosign + uses: sigstore/cosign-installer@v4.0.0 + + - name: Sign multi-arch manifest + run: cosign sign --yes "${IMAGE_NAME}@${{ steps.push-manifest.outputs.digest }}" - # Store build context - name: Add digest to build context run: | mkdir builds/ - digest="${{ steps.build-and-push.outputs.digest }}" + digest="${{ steps.push-manifest.outputs.digest }}" echo '${{ toJSON(matrix) }}' | jq --arg digest "$digest" '. +={"digest": $digest}' >> "builds/${{ matrix.key }}.json" - name: Upload build context diff --git a/src/docker_python_nodejs/build_matrix.py b/src/docker_python_nodejs/build_matrix.py index 6d8fd43..80e7e85 100644 --- a/src/docker_python_nodejs/build_matrix.py +++ b/src/docker_python_nodejs/build_matrix.py @@ -25,7 +25,32 @@ def _github_action_set_output(key: str, value: str) -> None: sys.exit(1) with Path(GITHUB_OUTPUT).open("a") as fp: - fp.write(f"{key}={value}") + fp.write(f"{key}={value}\n") + + +def _build_matrix_json(new_or_updated: list[BuildVersion]) -> str: + return json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" + + +def _build_arch_matrix_json(new_or_updated: list[BuildVersion]) -> str: + if not new_or_updated: + return "" + + include: list[dict[str, object]] = [] + for version in new_or_updated: + include.extend( + ( + dataclasses.asdict(version) + | { + "platform": platform, + "arch": platform.split("/")[1], + "runner": "ubuntu-24.04-arm" if platform == "linux/arm64" else "ubuntu-latest", + } + ) + for platform in version.platforms + ) + + return json.dumps({"include": include}) def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: @@ -33,7 +58,9 @@ def build_matrix(new_or_updated: list[BuildVersion], ci_event: str) -> None: logger.info("\n# Scheduled run with no new or updated versions. Doing nothing.") return - matrix = json.dumps({"include": [dataclasses.asdict(ver) for ver in new_or_updated]}) if new_or_updated else "" - _github_action_set_output("MATRIX", matrix) + matrix = _build_matrix_json(new_or_updated) + arch_matrix = _build_arch_matrix_json(new_or_updated) + _github_action_set_output("matrix", matrix) + _github_action_set_output("arch_matrix", arch_matrix) logger.info("\n# New or updated versions:") logger.info("Nothing" if not new_or_updated else "\n".join(version.key for version in new_or_updated)) diff --git a/tests/test_all.py b/tests/test_all.py index 433e113..d2390ae 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -8,6 +8,7 @@ import pytest import responses +from docker_python_nodejs.build_matrix import _build_arch_matrix_json from docker_python_nodejs.dockerfiles import render_dockerfile_with_context from docker_python_nodejs.readme import update_dynamic_readme from docker_python_nodejs.settings import BASE_PATH, DOCKERFILES_PATH @@ -289,3 +290,40 @@ def test_find_new_or_updated_with_digest() -> None: res = find_new_or_updated([new], {existing.key: existing}) assert len(res) == 0 + + +def test_build_arch_matrix_json(build_version: BuildVersion) -> None: + matrix = json.loads(_build_arch_matrix_json([build_version])) + + assert matrix == { + "include": [ + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/amd64", + "arch": "amd64", + "runner": "ubuntu-latest", + }, + { + "key": "python3.11-nodejs20", + "python": "3.11", + "python_canonical": "3.11.3", + "python_image": "3.11.3-trixie", + "nodejs": "20", + "nodejs_canonical": "20.2.0", + "distro": "trixie", + "platforms": ["linux/amd64", "linux/arm64"], + "digest": "", + "platform": "linux/arm64", + "arch": "arm64", + "runner": "ubuntu-24.04-arm", + }, + ], + }