ArchiveBox/.github/workflows/docker.yml

name: Build Docker image

on:
  workflow_dispatch:
  workflow_call:
  push:
    branches:
      - "**"
    tags:
      - 'v*'
  # pull_request:

env:
  DOCKERHUB_IMAGE: archivebox/archivebox
  GHCR_IMAGE: ghcr.io/archivebox/archivebox

permissions:
  contents: read
  packages: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    name: build ${{ matrix.platform }}
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - platform: linux/amd64
            runner: ubuntu-24.04
            cache_scope: docker-amd64
            artifact_name: digest-linux-amd64
          - platform: linux/arm64
            runner: ubuntu-24.04-arm
            cache_scope: docker-arm64
            artifact_name: digest-linux-arm64
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        # with:
        #   submodules: true
        #   fetch-depth: 1

      - name: Wait for released ArchiveBox deps on PyPI
        run: |
          python3 - <<'PY'
          import json
          import re
          import sys
          import time
          import tomllib
          import urllib.request

          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
          required = {}
          for dep in deps:
              for name in watched:
                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
                  if match:
                      required[name] = match.group(2)

          deadline = time.monotonic() + 300
          missing = required.copy()
          while missing and time.monotonic() < deadline:
              for name, version in list(missing.items()):
                  try:
                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
                          available = True
                  except Exception:
                      available = False
                  if available:
                      print(f"{name} {version} is available on PyPI")
                      missing.pop(name)
                  else:
                      print(f"{name} {version} is not available on PyPI yet")
              if missing:
                  time.sleep(10)

          if missing:
              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
              sys.exit(1)
          PY

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: latest
          install: true

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}

      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

      - name: Wait for published abx-dl image
        id: abx_dl_image
        shell: bash
        run: |
          set -Eeuo pipefail
          ABX_DL_VERSION="$(python3 - <<'PY'
          import re
          import tomllib

          with open("pyproject.toml", "rb") as f:
              deps = tomllib.load(f)["project"]["dependencies"]

          for dep in deps:
              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
              if match:
                  print(match.group(1))
                  break
          else:
              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
          PY
          )"
          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
          deadline=$((SECONDS + 1800))
          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
            if (( SECONDS >= deadline )); then
              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
              exit 1
            fi
            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
            sleep 30
          done

          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
          docker buildx imagetools inspect "${ABX_DL_IMAGE}"

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Collect Docker labels
        id: docker_meta
        shell: bash
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
          import tomllib

          with open("pyproject.toml", "rb") as f:
              print(tomllib.load(f)["project"]["version"])
          PY
          )"

          {
            echo 'labels<<EOF'
            echo "org.opencontainers.image.version=${VERSION}"
            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
            echo 'EOF'
            echo "version=${VERSION}"
          } >> "$GITHUB_OUTPUT"

          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"

      - name: Build and push digest
        id: docker_build
        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
            ${{ env.DOCKERHUB_IMAGE }}
            ${{ env.GHCR_IMAGE }}
          labels: ${{ steps.docker_meta.outputs.labels }}
          build-args: |
            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
          cache-from: type=gha,scope=${{ matrix.cache_scope }}
          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
          pull: true
          platforms: ${{ matrix.platform }}
          outputs: type=image,push-by-digest=true,name-canonical=true,push=true

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}

      - name: Validate pushed image version, commit, and size
        shell: bash
        run: |
          set -Eeuo pipefail
          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
          SHORT_SHA="${GITHUB_SHA::7}"

          DATA_DIR="$(mktemp -d)"
          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt

          python3 - "$IMAGE" <<'PY'
          import json
          import subprocess
          import sys

          image = sys.argv[1]
          limit = 780 * 1024 * 1024
          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
          if total > limit:
              raise SystemExit(f"{image} is over the compressed size limit")
          PY

      - name: Export digest
        shell: bash
        run: |
          set -Eeuo pipefail
          mkdir -p /tmp/digests
          digest="${{ steps.docker_build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"

      - name: Upload digest
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact_name }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  publish:
    name: publish multiarch tags
    runs-on: ubuntu-24.04
    needs:
      - build
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: latest
          install: true

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download digests
        uses: actions/download-artifact@v4
        with:
          path: /tmp/digests
          pattern: digest-*
          merge-multiple: true

      - name: Collect Docker tags
        id: docker_meta
        shell: bash
        run: |
          set -Eeuo pipefail
          VERSION="$(python3 - <<'PY'
          import tomllib

          with open("pyproject.toml", "rb") as f:
              print(tomllib.load(f)["project"]["version"])
          PY
          )"
          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
          SHORT_SHA="${GITHUB_SHA::12}"
          test -n "$BRANCH_TAG"
          test -n "$SHORT_SHA"

          {
            echo 'dockerhub_tags<<EOF'
            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
              echo "${DOCKERHUB_IMAGE}:latest"
            fi
            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
            echo "${DOCKERHUB_IMAGE}:${VERSION}"
            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo 'ghcr_tags<<EOF'
            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
              echo "${GHCR_IMAGE}:latest"
            fi
            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
            echo "${GHCR_IMAGE}:${VERSION}"
            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
            echo 'EOF'
            echo "version=${VERSION}"
            echo "branch_tag=${BRANCH_TAG}"
            echo "short_sha=${SHORT_SHA}"
          } >> "$GITHUB_OUTPUT"

          echo "[+] Publishing Docker Hub tags:"
          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
          fi
          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
          echo "[+] Publishing GHCR tags:"
          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
            printf '%s\n' "${GHCR_IMAGE}:latest"
          fi
          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"

      - name: Create Docker Hub manifest
        shell: bash
        run: |
          set -Eeuo pipefail
          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
          [[ "${#DIGESTS[@]}" -gt 0 ]]

          TAG_ARGS=()
          while IFS= read -r tag; do
            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"

          REFS=()
          for digest in "${DIGESTS[@]}"; do
            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
          done

          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"

      - name: Create GHCR manifest
        shell: bash
        run: |
          set -Eeuo pipefail
          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
          [[ "${#DIGESTS[@]}" -gt 0 ]]

          TAG_ARGS=()
          while IFS= read -r tag; do
            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"

          REFS=()
          for digest in "${DIGESTS[@]}"; do
            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
          done

          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"

      - name: Inspect published images
        shell: bash
        run: |
          set -Eeuo pipefail
          while IFS= read -r tag; do
            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
          while IFS= read -r tag; do
            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"

      - name: Update README
        uses: peter-evans/dockerhub-description@v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          repository: archivebox/archivebox