diff --git a/.ci/Debian12/Dockerfile b/.ci/Debian12/Dockerfile index fc756aac2..14c6f018c 100644 --- a/.ci/Debian12/Dockerfile +++ b/.ci/Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Debian13/Dockerfile b/.ci/Debian13/Dockerfile index bdecb56df..b597307c5 100644 --- a/.ci/Debian13/Dockerfile +++ b/.ci/Debian13/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Servatrice_Debian12/Dockerfile b/.ci/Servatrice_Debian12/Dockerfile index 321aa7c0f..f1a16843a 100644 --- a/.ci/Servatrice_Debian12/Dockerfile +++ b/.ci/Servatrice_Debian12/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu24.04/Dockerfile b/.ci/Ubuntu24.04/Dockerfile index 715997474..b9fb2b701 100644 --- a/.ci/Ubuntu24.04/Dockerfile +++ b/.ci/Ubuntu24.04/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/Ubuntu26.04/Dockerfile b/.ci/Ubuntu26.04/Dockerfile index 3166b7eb4..cc6528858 100644 --- a/.ci/Ubuntu26.04/Dockerfile +++ b/.ci/Ubuntu26.04/Dockerfile @@ -5,7 +5,6 @@ RUN apt-get update && \ build-essential \ ca-certificates \ ccache \ - clang-format \ cmake \ file \ g++ \ diff --git a/.ci/compile.sh b/.ci/compile.sh index 606b9d201..302972f30 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -17,7 +17,7 @@ # --dir sets the name of the build dir, default is "build" # --cmake-generator sets CMAKE_GENERATOR as used by cmake # --target-macos-version sets the min os version - only used for macOS builds -# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE +# uses env: BUILDTYPE MAKE_INSTALL MAKE_PACKAGE PACKAGE_TYPE PACKAGE_SUFFIX MAKE_SERVER MAKE_NO_CLIENT MAKE_TEST USE_CCACHE CCACHE_SIZE CCACHE_EVICTION_AGE BUILD_DIR CMAKE_GENERATOR TARGET_MACOS_VERSION MAKE_SANITIZE MAKE_COVERAGE VCPKG_TARGET_MACOS_VERSION # (correspond to args: --debug/--release --install --package --suffix --server --test --sanitize --coverage --ccache --dir ) # exitcode: 1 for failure, 3 for invalid arguments @@ -243,8 +243,14 @@ if [[ $RUNNER_OS == macOS ]]; then triplet_source="../vcpkg/triplets/community/$arch-osx.cmake" fi cp "$triplet_source" "$triplet_file" - echo "set(VCPKG_CMAKE_SYSTEM_VERSION $TARGET_MACOS_VERSION)" >>"$triplet_file" - echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $TARGET_MACOS_VERSION)" >>"$triplet_file" + # The triplet is what vcpkg hashes into the ABI hash of every port, so two legs that + # agree here reuse each other's binary cache. Targets 14, 15 and 26 on the same Xcode + # produce identical binaries when they share the lowest deployment target, so they all + # build against 14 to keep one vcpkg cache scope instead of three. + # Keep in sync with "vcpkg_scope" and "vcpkg_deployment_target" in desktop-build.yml. + vcpkg_target="${VCPKG_TARGET_MACOS_VERSION:-$TARGET_MACOS_VERSION}" + echo "set(VCPKG_CMAKE_SYSTEM_VERSION $vcpkg_target)" >>"$triplet_file" + echo "set(VCPKG_OSX_DEPLOYMENT_TARGET $vcpkg_target)" >>"$triplet_file" flags+=("-DVCPKG_OVERLAY_TRIPLETS=$triplets_dir") flags+=("-DVCPKG_HOST_TRIPLET=$triplet_version") flags+=("-DVCPKG_TARGET_TRIPLET=$triplet_version") diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 098aabc4f..1ce51afaa 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -184,7 +184,7 @@ jobs: uses: actions/cache/restore@v6 with: key: ccache-${{ matrix.distro }}${{ matrix.version }}${{ matrix.variant }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Build ${{ matrix.distro }} ${{ matrix.version }} Docker image" shell: bash @@ -267,7 +267,7 @@ jobs: uses: actions/cache/save@v6 with: key: ${{ steps.restore_ccache.outputs.cache-primary-key }} - path: ${{ env.CACHE }} + path: ${{ env.CACHE }}/.ccache - name: "Upload artifact" id: upload_artifact @@ -325,6 +325,8 @@ jobs: soc: Intel type: Release use_ccache: 1 + vcpkg_scope: 'vcpkg/macos13-x64-xcode26.3/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.3" - os: macOS @@ -341,6 +343,9 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over xcode: "26.6" - os: macOS @@ -357,6 +362,8 @@ jobs: soc: Apple type: Release use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: macOS @@ -370,6 +377,8 @@ jobs: soc: Apple type: Debug use_ccache: 1 + vcpkg_deployment_target: 14 + vcpkg_scope: 'vcpkg/macos14-arm64-xcode26.6/' xcode: "26.6" - os: Windows @@ -383,6 +392,8 @@ jobs: qt_version: 6.11.* qt_modules: qtimageformats qtmultimedia qtshadertools qtwebsockets type: Release + vcpkg_scope: 'vcpkg/windows10-x64-vs2026/' + vcpkg_upload: 1 # only one leg per vcpkg_scope hands its archives over name: ${{ matrix.os }} ${{ matrix.target }}${{ matrix.soc == 'Intel' && ' Intel' || '' }}${{ matrix.type == 'Debug' && ' Debug' || '' }} needs: configure @@ -480,8 +491,28 @@ jobs: id: vcpkg-cache uses: TAServers/vcpkg-cache@v3 with: + # Every entry is keyed by vcpkg ABI hash, and one hash set exists per platform, + # target and toolchain. The action restores every entry it finds under the + # prefix, not just the hashes vcpkg asks for, so with one shared vcpkg/ + # namespace every leg downloads all 66 archives of all 5 legs on every run. + # Each of those restores also refreshes the entry's last access date, which + # is why a dead ABI generation is never evicted by age. Naming the scope is + # what makes those generations identifiable, and prune-caches below able to + # delete them. + prefix: ${{ matrix.vcpkg_scope }} token: ${{ secrets.GITHUB_TOKEN }} + # The action only creates its directory when the prefix matched at least one + # entry, and it reads that directory unguarded when it saves at the end of the + # job. A scope nobody has written to yet therefore leaves the directory missing + # and fails the job, which is exactly the state every scope starts in. Make the + # directory up front so an empty scope is a cache miss instead of an error. + - name: "Create vcpkg cache directory" + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + run: mkdir -p "$CACHE_DIR" + # Uses environment variables, see compile.sh for more details - name: "Build Cockatrice" id: build @@ -501,9 +532,19 @@ jobs: PACKAGE_SUFFIX: '${{ matrix.package_suffix }}' TARGET_MACOS_VERSION: ${{ matrix.override_target }} USE_CCACHE: ${{ matrix.use_ccache }} + # Only pushes to the default branch may write the shared binary cache: the + # archives a run compiles are only as trustworthy as the code they were + # built from, and only merged code may populate the scope that master and + # release builds restore and link. Pull requests (same-repository ones + # included) restore the shared cache but never write it; with the source + # left at read the collect step below finds nothing on PRs, so no + # pull-request-scoped entry is ever saved that no later run could restore. + # Forks pushing to their own default branch resolve github.ref against + # their own repository, so they keep a working cache of their own there. VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},${{ case(github.ref == 'refs/heads/master', 'readwrite', 'read') }}" VCPKG_DISABLE_METRICS: 1 VCPKG_FEATURE_FLAGS: dependencygraph + VCPKG_TARGET_MACOS_VERSION: ${{ matrix.vcpkg_deployment_target }} run: .ci/compile.sh --server --test --vcpkg # CPack reports a failed makensis run as "Problem running NSIS command" @@ -517,6 +558,63 @@ jobs: shell: bash run: find . -name NSISOutput.log -exec cat {} + + # A master run hands over the archives it had to build itself so that + # vcpkg-cache-import.yml, which runs in the default branch scope, can save them + # where later runs can restore them. Pull requests hand over nothing: their cache + # entries would be scoped to refs/pull//merge and stay unreadable for every + # other run, so writing them only burns the repository's quota. + # vcpkg_upload keeps this to one leg per scope: macOS 14, 15 and 26 share a scope + # by design, and all three rebuild the same archives, so uploading from each of them + # would publish three identical 269 MB artifacts under one artifact name and make + # the importer spawn a save job per copy. + - name: "Collect new vcpkg entries" + id: vcpkg_new + if: github.ref == 'refs/heads/master' && matrix.vcpkg_upload == 1 + shell: bash + env: + CACHE_DIR: '${{ steps.vcpkg-cache.outputs.path }}' + GH_TOKEN: ${{ github.token }} + VCPKG_SCOPE: '${{ matrix.vcpkg_scope }}' + run: | + # The archive name is the ABI hash, so a missing key is exactly what vcpkg had + # to compile from source in this job. + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/caches?ref=refs/heads/master&key=$VCPKG_SCOPE&per_page=100" \ + --jq '.actions_caches[].key' | sort -u >"$RUNNER_TEMP/master-keys.txt" + find "$CACHE_DIR" -type f -name '*.zip' -exec basename {} .zip \; | + sed "s|^|$VCPKG_SCOPE|" | sort -u >"$RUNNER_TEMP/local-keys.txt" + comm -23 "$RUNNER_TEMP/local-keys.txt" "$RUNNER_TEMP/master-keys.txt" >"$RUNNER_TEMP/new-keys.txt" + + mkdir -p "$RUNNER_TEMP/vcpkg-new" + : >"$RUNNER_TEMP/vcpkg-new/keys.txt" + while read -r key; do + hash="${key##*/}" + archive="$(find "$CACHE_DIR" -type f -name "$hash.zip" -print -quit)" + dir="$RUNNER_TEMP/vcpkg-new/${hash:0:2}" # vcpkg's own two character fanout + mkdir -p "$dir" + cp "$archive" "$dir/" + echo "$key ${hash:0:2}/$hash.zip" >>"$RUNNER_TEMP/vcpkg-new/keys.txt" + done <"$RUNNER_TEMP/new-keys.txt" + + count="$(wc -l <"$RUNNER_TEMP/new-keys.txt")" + echo "vcpkg scope $VCPKG_SCOPE: $count entries to import" + echo "count=$count" >>"$GITHUB_OUTPUT" + echo "slug=$(sed -e 's|^vcpkg/||' -e 's|/$||' -e 's|/|-|g' <<<"$VCPKG_SCOPE")" >>"$GITHUB_OUTPUT" + + # archive: false accepts a single file and nothing else, and a scope hands over one + # archive per ABI hash vcpkg had to rebuild, which is 23 files on a cold runner. + # Zipping the directory keeps the fanout at the artifact root, which is where + # vcpkg-cache-import.yml looks for keys.txt once download-artifact unpacks it. + - name: "Upload new vcpkg entries" + # An empty count means the step above did not run at all, on master and on forks + if: steps.vcpkg_new.outputs.count != '' && steps.vcpkg_new.outputs.count != '0' + uses: actions/upload-artifact@v7 + with: + if-no-files-found: error # a count above zero with nothing staged is a bug + name: vcpkg-new-${{ steps.vcpkg_new.outputs.slug }} + path: ${{ runner.temp }}/vcpkg-new + retention-days: 1 + # Delete used cache to emulate a ccache update. See https://github.com/actions/cache/issues/342 - name: "[macOS] Delete remote compiler cache (ccache)" if: matrix.os == 'macOS' && matrix.use_ccache == 1 && github.ref == 'refs/heads/master' && steps.restore_ccache.outputs.cache-hit @@ -629,3 +727,67 @@ jobs: BUILD_PATH: ${{ steps.build.outputs.path }} GH_TOKEN: ${{ github.token }} run: gh attestation verify "$BUILD_PATH" --repo Cockatrice/Cockatrice + + prune-caches: + name: Prune caches + # Only master writes caches at all, so only master may clean them up too + if: github.ref == 'refs/heads/master' + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + continue-on-error: true # freeing space must never fail a build + + steps: + # The repository holds 10Gi of cache against entries that want more, and GitHub + # evicts the least recently used entry whenever that is exceeded. The vcpkg + # entries are the reason that never happens: the restore above touches every + # entry under the prefix on every run, so a dead ABI generation keeps its access + # date fresh forever and age based eviction can never reach it. Deleting by age + # alone cannot fix that, only knowing which scope an entry belongs to can. + - name: "Delete unusable cache entries" + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Must list every vcpkg_scope of the build-vcpkg matrix above, or the entries of + # a scope that is still in use get deleted on the next run. + LIVE_VCPKG_SCOPES: 'vcpkg/macos13-x64-xcode26.3/ vcpkg/macos14-arm64-xcode26.6/ vcpkg/windows10-x64-vs2026/' + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/caches?per_page=100" \ + --jq '.actions_caches[] | [.id, .ref, .key, .last_accessed_at] | @tsv' >"$RUNNER_TEMP/caches.tsv" + + cutoff="$(( $(date -u +%s) - 7 * 24 * 3600 ))" + live=" ${LIVE_VCPKG_SCOPES} " + deleted=0 + while IFS=$'\t' read -r id ref key accessed; do + [[ -n $id ]] || continue + # Only ever touch the key shapes this workflow creates, so that an entry of + # some other workflow can never be deleted by accident + case "$key" in + vcpkg/* | ccache-* | Qt*) ;; + *) continue ;; + esac + + reason="" + if [[ $ref != refs/heads/master ]]; then + reason="scoped to $ref" + elif [[ $key == vcpkg/* ]]; then + # "/", cut the hash off the end to get the scope + scope="vcpkg/${key#vcpkg/}" + scope="${scope%/*}/" + if [[ $live != *" $scope "* ]]; then + reason="no matrix leg uses $scope" + fi + fi + if [[ -z $reason ]]; then + stamp="$(date -u -d "$accessed" +%s 2>/dev/null || echo 0)" + if [[ $stamp -ne 0 && $stamp -lt $cutoff ]]; then + reason="unused since $accessed" + fi + fi + + if [[ -n $reason ]]; then + echo "::notice::deleting $key, $reason" + gh api -X DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" + deleted=$((deleted + 1)) + fi + done <"$RUNNER_TEMP/caches.tsv" + + echo "::notice::deleted $deleted of $(wc -l <"$RUNNER_TEMP/caches.tsv") cache entries" diff --git a/.github/workflows/vcpkg-cache-import.yml b/.github/workflows/vcpkg-cache-import.yml new file mode 100644 index 000000000..3cc7b2203 --- /dev/null +++ b/.github/workflows/vcpkg-cache-import.yml @@ -0,0 +1,121 @@ +name: Import vcpkg cache + +# After a runner image update changes every port's ABI hash, the first master run +# rebuilds protobuf, openssl and abseil from source and saves them into the shared +# cache, where every later run can restore them. +# +# Only master pushes are imported here: the collect step, keys.txt and the zip +# contents all come from the workflow run being imported, so importing from an +# unmerged pull request would let anyone who can push a branch — or a bot actor +# such as Dependabot, whose PRs are same-repo — promote arbitrary binaries into +# the scope that master and tag builds restore and link into signed releases. +# Forks build untrusted code and must never reach that scope either. +# +# repository_dispatch is not used on purpose: it would also resolve to the default branch, +# but it counts as a low-trust trigger and may only write there when it declares a +# write-capable cache-mode, which puts it in the same position as workflow_run. + +permissions: + actions: write # needed to save entries into the default branch's cache scope + contents: read + +on: + workflow_run: + workflows: ["Build Desktop"] + types: [completed] + +# Every import writes the same scopes, so let them queue instead of racing each other +concurrency: + group: vcpkg-cache-import + cancel-in-progress: false + +jobs: + list: + name: List new vcpkg entries + # Only a successful push to the default branch has archives worth importing. + # Only code that landed on master may populate the scope master runs restore. + # Bot actors are excluded on top: their runs must never promote binaries into + # the shared scope even if they somehow target the default branch. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'master' && + github.event.workflow_run.head_repository.full_name == github.repository && + !endsWith(github.event.workflow_run.actor.login, '[bot]') && + !endsWith(github.event.workflow_run.triggering_actor.login, '[bot]') + runs-on: ubuntu-slim # https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md + # workflow_run is a low-trust trigger, so cache access is read-only by default even + # though this run resolves to the default branch. Assert below that this took effect, + # because a save under read-only access only warns and would import nothing silently. + cache-mode: write + outputs: + entries: ${{ steps.scan.outputs.entries }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: vcpkg-new-* + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Assert write access" + shell: bash + run: | + if [[ $ACTIONS_CACHE_MODE != *write* ]]; then + echo "::error::expected a write-capable cache, got '$ACTIONS_CACHE_MODE'" + exit 1 + fi + + - name: "List cache entries" + id: scan + shell: bash + working-directory: incoming + run: | + entries="$(find . -name keys.txt -type f -exec sh -c ' + for manifest; do + artifact="$(basename "$(dirname "$manifest")")" + while read -r key path; do + printf "{\"artifact\": \"%s\", \"key\": \"%s\", \"path\": \"%s\"}\n" \ + "$artifact" "$key" "$path" + done <"$manifest" + done + ' _ {} +)" + [[ -n $entries ]] || exit 0 + # Compact, because a multi line value cannot be written to GITHUB_OUTPUT + entries="$(jq -cs '.' <<<"$entries")" + echo "entries=$entries" >>"$GITHUB_OUTPUT" + echo "::notice::importing $(jq length <<<"$entries") vcpkg entries" + + save: + name: "Save ${{ matrix.entry.key }}" + needs: list + if: needs.list.outputs.entries + runs-on: ubuntu-slim + cache-mode: write + strategy: + # One job per archive, they run in parallel and each only uploads a single file + fail-fast: false + max-parallel: 20 + matrix: + entry: ${{ fromJSON(needs.list.outputs.entries) }} + + steps: + - name: "Download new vcpkg entries" + uses: actions/download-artifact@v8 + with: + pattern: ${{ matrix.entry.artifact }} + path: incoming + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Save vcpkg entry" + uses: actions/cache/save@v6 + with: + # The payload is a Windows or macOS archive while the upload runs on Linux, and + # without this the entry would be stamped as a Linux cache that neither of them + # is allowed to restore. + enableCrossOsArchive: true + key: ${{ matrix.entry.key }} + path: incoming/${{ matrix.entry.path }} \ No newline at end of file