synapse/.github/workflows/release-artifacts.yml

# GitHub actions workflow which builds the release artifacts.

name: Build release artifacts

on:
  # we build on PRs and develop to (hopefully) get early warning
  # of things breaking (but only build one set of debs). PRs skip
  # building wheels on macOS & ARM.
  pull_request:
  push:
    branches: ["develop", "release-*"]

    # we do the full build on tags.
    tags: ["v*"]
  merge_group:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: write

jobs:
  get-distros:
    name: "Calculate list of debian distros"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.x'
      - id: set-distros
        run: |
          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
          dists='["debian:sid"]'
          if [[ $GITHUB_REF == refs/tags/* ]]; then
              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
          fi
          echo "distros=$dists" >> "$GITHUB_OUTPUT"          
    # map the step outputs to job outputs
    outputs:
      distros: ${{ steps.set-distros.outputs.distros }}

  # now build the packages with a matrix build.
  build-debs:
    needs: get-distros
    name: "Build .deb packages"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        distro: ${{ fromJson(needs.get-distros.outputs.distros) }}

    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          path: src

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
        with:
          install: true

      - name: Set up docker layer caching
        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-            

      - name: Set up python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.x'

      - name: Build the packages
        # see https://github.com/docker/build-push-action/issues/252
        # for the cache magic here
        run: |
          ./src/scripts-dev/build_debian_packages.py \
            --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
            --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
            --docker-build-arg=--progress=plain \
            --docker-build-arg=--load \
            "${{ matrix.distro }}"
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache          

      - name: Artifact name
        id: artifact-name
        # We can't have colons in the upload name of the artifact, so we convert
        # e.g. `debian:sid` to `sid`.
        env:
          DISTRO: ${{ matrix.distro }}
        run: |
          echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT"          

      - name: Upload debs as artifacts
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
          path: debs/*

  build-wheels:
    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-22.04, macos-13]
        arch: [x86_64, aarch64]
        # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
        # It is not read by the rest of the workflow.
        is_pr:
          - ${{ startsWith(github.ref, 'refs/pull/') }}

        exclude:
          # Don't build macos wheels on PR CI.
          - is_pr: true
            os: "macos-13"
          # Don't build aarch64 wheels on mac.
          - os: "macos-13"
            arch: aarch64
          # Don't build aarch64 wheels on PR CI.
          - is_pr: true
            arch: aarch64

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          # setup-python@v4 doesn't impose a default python version. Need to use 3.x
          # here, because `python` on osx points to Python 2.7.
          python-version: "3.x"

      - name: Install cibuildwheel
        run: python -m pip install cibuildwheel==2.23.0

      - name: Set up QEMU to emulate aarch64
        if: matrix.arch == 'aarch64'
        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
        with:
          platforms: arm64

      - name: Build aarch64 wheels
        if: matrix.arch == 'aarch64'
        run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV

      - name: Only build a single wheel on PR
        if: startsWith(github.ref, 'refs/pull/')
        run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV

      - name: Build wheels
        run: python -m cibuildwheel --output-dir wheelhouse
        env:
          # Skip testing for platforms which various libraries don't have wheels
          # for, and so need extra build deps.
          CIBW_TEST_SKIP: pp3*-* *i686* *musl*
          # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
          CARGO_NET_GIT_FETCH_WITH_CLI: true
          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI

      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: Wheel-${{ matrix.os }}-${{ matrix.arch }}
          path: ./wheelhouse/*.whl

  build-sdist:
    name: Build sdist
    runs-on: ubuntu-latest
    if: ${{ !startsWith(github.ref, 'refs/pull/') }}

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.10'

      - run: pip install build

      - name: Build sdist
        run: python -m build --sdist

      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: Sdist
          path: dist/*.tar.gz


  # if it's a tag, create a release and attach the artifacts to it
  attach-assets:
    name: "Attach assets to release"
    if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
    needs:
      - build-debs
      - build-wheels
      - build-sdist
    runs-on: ubuntu-latest
    steps:
      - name: Download all workflow run artifacts
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
      - name: Build a tarball for the debs
        # We need to merge all the debs uploads into one folder, then compress
        # that.
        run: |
          mkdir debs
          mv debs*/* debs/
          tar -cvJf debs.tar.xz debs          
      - name: Attach to release
        # Pinned to work around https://github.com/softprops/action-gh-release/issues/445
        uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v0.1.15
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: |
            Sdist/*
            Wheel*/*
            debs.tar.xz            
          # if it's not already published, keep the release as a draft.
          draft: true
          # mark it as a prerelease if the tag contains 'rc'.
          prerelease: ${{ contains(github.ref, 'rc') }}