From 25c85cf0bb818eced0a19ae09f785abe4a86aa47 Mon Sep 17 00:00:00 2001 From: BruebachL <44814898+BruebachL@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:14:16 +0200 Subject: [PATCH] [CI] Key the vcpkg binary cache per ABI and let pull requests heal it (#7377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [CI] Key the vcpkg binary cache per ABI and let pull requests heal it The cache had no prefix, and TAServers/vcpkg-cache restores every entry it finds under the prefix rather than only the hashes vcpkg asks for. With all legs sharing the single vcpkg/ namespace, each of the five of them downloads all 66 archives of all five on every run. Each restore also refreshes the entry's last access date, so a dead ABI generation stays fresh forever and GitHub's least recently used eviction can never reach it. Name the scope in the key after the platform, target and toolchain it belongs to, so a leg restores only its own archives. macOS 14, 15 and 26 on the same Xcode build identical binaries when they share the lowest deployment target, so point all three at 14 and let them share one scope instead of three. A pull request cannot write the shared cache either. Its entries are scoped to refs/pull//merge, so nothing it saves is ever readable again, and letting it write anyway only burns the repository's quota, which is what the previous attempt at this did. Keep the build read-only and hand the archives it had to build itself to a new workflow running in the default branch scope, which saves them where master can restore them. Forks stay read-only, they are the one case whose archives must not be trusted. Because the prefix changes every key, add a job that deletes the entries of retired scopes, which is the one-time clean up of the current flat namespace, and the per-run ones pull requests leave behind, which age based eviction also never reaches. Also pin the Linux ccache cache to its own subdirectory, since the directory is shared with the docker build context, and drop the clang-format install from the build images, which format nothing at runtime. * [CI] Create the vcpkg cache directory when a scope is still empty * [CI] Let a pull request write the vcpkg archives it builds * [CI] Upload the vcpkg handover as one archive instead of loose files * [CI] Hand the vcpkg cache over from one leg per scope * [CI] Gate vcpkg cache writes to master pushes --------- Co-authored-by: Lukas Brübach --- .ci/Debian12/Dockerfile | 1 - .ci/Debian13/Dockerfile | 1 - .ci/Servatrice_Debian12/Dockerfile | 1 - .ci/Ubuntu24.04/Dockerfile | 1 - .ci/Ubuntu26.04/Dockerfile | 1 - .ci/compile.sh | 12 +- .github/workflows/desktop-build.yml | 166 ++++++++++++++++++++++- .github/workflows/vcpkg-cache-import.yml | 121 +++++++++++++++++ 8 files changed, 294 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/vcpkg-cache-import.yml diff --git a/.ci/Debian12/Dockerfile b/.ci/Debian12/Dockerfile index fc756aac2..14c6f018c 100644 --- a/.ci/Debian12/Dockerfile +++ b/.ci/Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Debian13/Dockerfile b/.ci/Debian13/Dockerfile index bdecb56df..b597307c5 100644 --- a/.ci/Debian13/Dockerfile +++ b/.ci/Debian13/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Servatrice_Debian12/Dockerfile b/.ci/Servatrice_Debian12/Dockerfile index 321aa7c0f..f1a16843a 100644 --- a/.ci/Servatrice_Debian12/Dockerfile +++ b/.ci/Servatrice_Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu24.04/Dockerfile b/.ci/Ubuntu24.04/Dockerfile index 715997474..b9fb2b701 100644 --- a/.ci/Ubuntu24.04/Dockerfile +++ b/.ci/Ubuntu24.04/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu26.04/Dockerfile b/.ci/Ubuntu26.04/Dockerfile index 3166b7eb4..cc6528858 100644 --- a/.ci/Ubuntu26.04/Dockerfile +++ b/.ci/Ubuntu26.04/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/compile.sh b/.ci/compile.sh index 606b9d201..302972f30 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -17,7 +17,7 @@ # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE VCPKG_TARGET_MACOS_VERSION # (correspond to args: --debug/--release --install --package --suffix --server --test --sanitize --coverage --ccache --dir ) # exitcode: 1 for failure, 3 for invalid arguments @@ -243,8 +243,14 @@ if [[ $RUNNER_OS == macOS ]]; then triplet_source="../vcpkg/triplets/community/$arch-osx.cmake" fi cp "$triplet_source" "$triplet_file" - echo "set(VCPKG_CMAKE_SYSTEM_VERSION $TARGET_MACOS_VERSION)" >>"$triplet_file" - echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $TARGET_MACOS_VERSION)" >>"$triplet_file" + # The triplet is what vcpkg hashes into the ABI hash of every port, so two legs that + # agree here reuse each other's binary cache. Targets 14, 15 and 26 on the same Xcode + # produce identical binaries when they share the lowest deployment target, so they all + # build against 14 to keep one vcpkg cache scope instead of three. + # Keep in sync with "vcpkg_scope" and "vcpkg_deployment_target" in desktop-build.yml. + vcpkg_target="${VCPKG_TARGET_MACOS_VERSION:-$TARGET_MACOS_VERSION}" + echo "set(VCPKG_CMAKE_SYSTEM_VERSION $vcpkg_target)" >>"$triplet_file" + echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $vcpkg_target)" >>"$triplet_file" flags+=("-DVCPKG_OVERLAY_TRIPLETS=$triplets_dir") flags+=("-DVCPKG_HOST_TRIPLET=$triplet_version") flags+=("-DVCPKG_TARGET_TRIPLET=$triplet_version") diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 098aabc4f..1ce51afaa 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -184,7 +184,7 @@ jobs: uses: actions/cache/restore@v6 with: key: ccache-${{ matrix.distro }}${{ matrix.version }}${{ matrix.variant }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Build ${{ matrix.distro }} ${{ matrix.version }} Docker image" shell: bash @@ -267,7 +267,7 @@ jobs: uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Upload artifact" id: upload_artifact @@ -325,6 +325,8 @@ jobs: soc: Intel type: Release use_ccache: 1 + vcpkg_scope: 'vcpkg/macos13-x64-xcode26.3/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.3" - os: macOS @@ -341,6 +343,9 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.6" - os: macOS @@ -357,6 +362,8 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: macOS @@ -370,6 +377,8 @@ jobs: soc: Apple type: Debug use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: Windows @@ -383,6 +392,8 @@ jobs: qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets type: Release + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} needs: configure @@ -480,8 +491,28 @@ jobs: id: vcpkg-cache uses: TAServers/vcpkg-cache@v3 with: + # Every entry is keyed by vcpkg ABI hash, and one hash set exists per platform, + # target and toolchain. The action restores every entry it finds under the + # prefix, not just the hashes vcpkg asks for, so with one shared vcpkg/ + # namespace every leg downloads all 66 archives of all 5 legs on every run. + # Each of those restores also refreshes the entry's last access date, which + # is why a dead ABI generation is never evicted by age. Naming the scope is + # what makes those generations identifiable, and prune-caches below able to + # delete them. + prefix: ${{ matrix.vcpkg_scope }} token: ${{ secrets.GITHUB_TOKEN }} + # The action only creates its directory when the prefix matched at least one + # entry, and it reads that directory unguarded when it saves at the end of the + # job. A scope nobody has written to yet therefore leaves the directory missing + # and fails the job, which is exactly the state every scope starts in. Make the + # directory up front so an empty scope is a cache miss instead of an error. + - name: "Create vcpkg cache directory" + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + run: mkdir -p "$CACHE_DIR" + # Uses environment variables, see compile.sh for more details - name: "Build Cockatrice" id: build @@ -501,9 +532,19 @@ jobs: PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' TARGET_MACOS_VERSION: ${{ matrix.override_target }} USE_CCACHE: ${{ matrix.use_ccache }} + # Only pushes to the default branch may write the shared binary cache: the + # archives a run compiles are only as trustworthy as the code they were + # built from, and only merged code may populate the scope that master and + # release builds restore and link. Pull requests (same-repository ones + # included) restore the shared cache but never write it; with the source + # left at read the collect step below finds nothing on PRs, so no + # pull-request-scoped entry is ever saved that no later run could restore. + # Forks pushing to their own default branch resolve github.ref against + # their own repository, so they keep a working cache of their own there. VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master', 'readwrite', 'read') }}" VCPKG_DISABLE_METRICS: 1 VCPKG_FEATURE_FLAGS: dependencygraph + VCPKG_TARGET_MACOS_VERSION: ${{ matrix.vcpkg_deployment_target }} run: .ci/compile.sh --server --test --vcpkg # CPack reports a failed makensis run as "Problem running NSIS command" @@ -517,6 +558,63 @@ jobs: shell: bash run: find . -name NSISOutput.log -exec cat {} + + # A master run hands over the archives it had to build itself so that + # vcpkg-cache-import.yml, which runs in the default branch scope, can save them + # where later runs can restore them. Pull requests hand over nothing: their cache + # entries would be scoped to refs/pull//merge and stay unreadable for every + # other run, so writing them only burns the repository's quota. + # vcpkg_upload keeps this to one leg per scope: macOS 14, 15 and 26 share a scope + # by design, and all three rebuild the same archives, so uploading from each of them + # would publish three identical 269 MB artifacts under one artifact name and make + # the importer spawn a save job per copy. + - name: "Collect new vcpkg entries" + id: vcpkg_new + if: github.ref == 'refs/heads/master' && matrix.vcpkg_upload == 1 + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + GH_TOKEN: ${{ github.token }} + VCPKG_SCOPE: '${{ matrix.vcpkg_scope }}' + run: | + # The archive name is the ABI hash, so a missing key is exactly what vcpkg had + # to compile from source in this job. + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/caches?ref=refs/heads/master&key=$VCPKG_SCOPE&per_page=100" \ + --jq '.actions_caches[].key' | sort -u >"$RUNNER_TEMP/master-keys.txt" + find "$CACHE_DIR" -type f -name '*.zip' -exec basename {} .zip \; | + sed "s|^|$VCPKG_SCOPE|" | sort -u >"$RUNNER_TEMP/local-keys.txt" + comm -23 "$RUNNER_TEMP/local-keys.txt" "$RUNNER_TEMP/master-keys.txt" >"$RUNNER_TEMP/new-keys.txt" + + mkdir -p "$RUNNER_TEMP/vcpkg-new" + : >"$RUNNER_TEMP/vcpkg-new/keys.txt" + while read -r key; do + hash="${key##*/}" + archive="$(find "$CACHE_DIR" -type f -name "$hash.zip" -print -quit)" + dir="$RUNNER_TEMP/vcpkg-new/${hash:0:2}" # vcpkg's own two character fanout + mkdir -p "$dir" + cp "$archive" "$dir/" + echo "$key ${hash:0:2}/$hash.zip" >>"$RUNNER_TEMP/vcpkg-new/keys.txt" + done <"$RUNNER_TEMP/new-keys.txt" + + count="$(wc -l <"$RUNNER_TEMP/new-keys.txt")" + echo "vcpkg scope $VCPKG_SCOPE: $count entries to import" + echo "count=$count" >>"$GITHUB_OUTPUT" + echo "slug=$(sed -e 's|^vcpkg/||' -e 's|/$||' -e 's|/|-|g' <<<"$VCPKG_SCOPE")" >>"$GITHUB_OUTPUT" + + # archive: false accepts a single file and nothing else, and a scope hands over one + # archive per ABI hash vcpkg had to rebuild, which is 23 files on a cold runner. + # Zipping the directory keeps the fanout at the artifact root, which is where + # vcpkg-cache-import.yml looks for keys.txt once download-artifact unpacks it. + - name: "Upload new vcpkg entries" + # An empty count means the step above did not run at all, on master and on forks + if: steps.vcpkg_new.outputs.count != '' && steps.vcpkg_new.outputs.count != '0' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error # a count above zero with nothing staged is a bug + name: vcpkg-new-${{ steps.vcpkg_new.outputs.slug }} + path: ${{ runner.temp }}/vcpkg-new + retention-days: 1 + # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "[macOS] Delete remote compiler cache (ccache)" if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit @@ -629,3 +727,67 @@ jobs: BUILD_PATH: ${{ steps.build.outputs.path }} GH_TOKEN: ${{ github.token }} run: gh attestation verify "$BUILD_PATH" --repo Cockatrice/Cockatrice + + prune-caches: + name: Prune caches + # Only master writes caches at all, so only master may clean them up too + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + continue-on-error: true # freeing space must never fail a build + + steps: + # The repository holds 10Gi of cache against entries that want more, and GitHub + # evicts the least recently used entry whenever that is exceeded. The vcpkg + # entries are the reason that never happens: the restore above touches every + # entry under the prefix on every run, so a dead ABI generation keeps its access + # date fresh forever and age based eviction can never reach it. Deleting by age + # alone cannot fix that, only knowing which scope an entry belongs to can. + - name: "Delete unusable cache entries" + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Must list every vcpkg_scope of the build-vcpkg matrix above, or the entries of + # a scope that is still in use get deleted on the next run. + LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ + --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" + + cutoff="$(( $(date -u +%s) - 7 * 24 * 3600 ))" + live=" ${LIVE_VCPKG_SCOPES} " + deleted=0 + while IFS=$'\t' read -r id ref key accessed; do + [[ -n $id ]] || continue + # Only ever touch the key shapes this workflow creates, so that an entry of + # some other workflow can never be deleted by accident + case "$key" in + vcpkg/* | ccache-* | Qt*) ;; + *) continue ;; + esac + + reason="" + if [[ $ref != refs/heads/master ]]; then + reason="scoped to $ref" + elif [[ $key == vcpkg/* ]]; then + # "/", cut the hash off the end to get the scope + scope="vcpkg/${key#vcpkg/}" + scope="${scope%/*}/" + if [[ $live != *" $scope "* ]]; then + reason="no matrix leg uses $scope" + fi + fi + if [[ -z $reason ]]; then + stamp="$(date -u -d "$accessed" +%s 2>/dev/null || echo 0)" + if [[ $stamp -ne 0 && $stamp -lt $cutoff ]]; then + reason="unused since $accessed" + fi + fi + + if [[ -n $reason ]]; then + echo "::notice::deleting $key, $reason" + gh api -X DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" + deleted=$((deleted + 1)) + fi + done <"$RUNNER_TEMP/caches.tsv" + + echo "::notice::deleted $deleted of $(wc -l <"$RUNNER_TEMP/caches.tsv") cache entries" diff --git a/.github/workflows/vcpkg-cache-import.yml b/.github/workflows/vcpkg-cache-import.yml new file mode 100644 index 000000000..3cc7b2203 --- /dev/null +++ b/.github/workflows/vcpkg-cache-import.yml @@ -0,0 +1,121 @@ +name: Import vcpkg cache + +# After a runner image update changes every port's ABI hash, the first master run +# rebuilds protobuf, openssl and abseil from source and saves them into the shared +# cache, where every later run can restore them. +# +# Only master pushes are imported here: the collect step, keys.txt and the zip +# contents all come from the workflow run being imported, so importing from an +# unmerged pull request would let anyone who can push a branch — or a bot actor +# such as Dependabot, whose PRs are same-repo — promote arbitrary binaries into +# the scope that master and tag builds restore and link into signed releases. +# Forks build untrusted code and must never reach that scope either. +# +# repository_dispatch is not used on purpose: it would also resolve to the default branch, +# but it counts as a low-trust trigger and may only write there when it declares a +# write-capable cache-mode, which puts it in the same position as workflow_run. + +permissions: + actions: write # needed to save entries into the default branch's cache scope + contents: read + +on: + workflow_run: + workflows: ["Build Desktop"] + types: [completed] + +# Every import writes the same scopes, so let them queue instead of racing each other +concurrency: + group: vcpkg-cache-import + cancel-in-progress: false + +jobs: + list: + name: List new vcpkg entries + # Only a successful push to the default branch has archives worth importing. + # Only code that landed on master may populate the scope master runs restore. + # Bot actors are excluded on top: their runs must never promote binaries into + # the shared scope even if they somehow target the default branch. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'master' && + github.event.workflow_run.head_repository.full_name == github.repository && + !endsWith(github.event.workflow_run.actor.login, '[bot]') && + !endsWith(github.event.workflow_run.triggering_actor.login, '[bot]') + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + # workflow_run is a low-trust trigger, so cache access is read-only by default even + # though this run resolves to the default branch. Assert below that this took effect, + # because a save under read-only access only warns and would import nothing silently. + cache-mode: write + outputs: + entries: ${{ steps.scan.outputs.entries }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: vcpkg-new-* + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Assert write access" + shell: bash + run: | + if [[ $ACTIONS_CACHE_MODE != *write* ]]; then + echo "::error::expected a write-capable cache, got '$ACTIONS_CACHE_MODE'" + exit 1 + fi + + - name: "List cache entries" + id: scan + shell: bash + working-directory: incoming + run: | + entries="$(find . -name keys.txt -type f -exec sh -c ' + for manifest; do + artifact="$(basename "$(dirname "$manifest")")" + while read -r key path; do + printf "{\"artifact\": \"%s\", \"key\": \"%s\", \"path\": \"%s\"}\n" \ + "$artifact" "$key" "$path" + done <"$manifest" + done + ' _ {} +)" + [[ -n $entries ]] || exit 0 + # Compact, because a multi line value cannot be written to GITHUB_OUTPUT + entries="$(jq -cs '.' <<<"$entries")" + echo "entries=$entries" >>"$GITHUB_OUTPUT" + echo "::notice::importing $(jq length <<<"$entries") vcpkg entries" + + save: + name: "Save ${{ matrix.entry.key }}" + needs: list + if: needs.list.outputs.entries + runs-on: ubuntu-slim + cache-mode: write + strategy: + # One job per archive, they run in parallel and each only uploads a single file + fail-fast: false + max-parallel: 20 + matrix: + entry: ${{ fromJSON(needs.list.outputs.entries) }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: ${{ matrix.entry.artifact }} + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Save vcpkg entry" + uses: actions/cache/save@v6 + with: + # The payload is a Windows or macOS archive while the upload runs on Linux, and + # without this the entry would be stamped as a Linux cache that neither of them + # is allowed to restore. + enableCrossOsArchive: true + key: ${{ matrix.entry.key }} + path: incoming/${{ matrix.entry.path }} \ No newline at end of file