From 72a85b58cf1c3d188aacb92b295d57a4b45a5524 Mon Sep 17 00:00:00 2001 From: Bruno Alexandre Rosa <1791393+brunoalr@users.noreply.github.com> Date: Sun, 4 Jan 2026 01:00:05 +0100 Subject: [PATCH] ci: make fat qt libs thin (#6281) * ci: strip fat qt binaries * parallelize * cache thin qt * print libs * change qt install dir in the action * move qt install logic to separate job * lookup only * debug: show contents of QTDIR * enableCrossOsArchive also when saving * check one dir up * change install dir * keep debugging * try deleting cache * force delete cache * pass gh_token * pass missing params * use api * change cache key, disable cross os archive * move job directly to steps * add comments * set cache param directly * address comments * fixup * Update .ci/thin_macos_qtlib.sh * resolve qt version * move resolution to separate script * use single line for run: * improve error handling in new scripts --------- Co-authored-by: ebbit1q --- .ci/compile.sh | 12 +++++++ .ci/resolve_latest_aqt_qt_version.sh | 40 +++++++++++++++++++++ .ci/thin_macos_qtlib.sh | 25 +++++++++++++ .github/workflows/desktop-build.yml | 54 +++++++++++++++++++++++----- 4 files changed, 123 insertions(+), 8 deletions(-) create mode 100755 .ci/resolve_latest_aqt_qt_version.sh create mode 100755 .ci/thin_macos_qtlib.sh diff --git a/.ci/compile.sh b/.ci/compile.sh index 0af79868b..424527f96 100755 --- a/.ci/compile.sh +++ b/.ci/compile.sh @@ -156,6 +156,18 @@ function ccachestatsverbose() { # Compile if [[ $RUNNER_OS == macOS ]]; then + # QTDIR is needed for macOS since we actually only use the cached thin Qt binaries instead of the install-qt-action, + # which sets a few environment variables + if QTDIR=$(find "$GITHUB_WORKSPACE/Qt" -depth -maxdepth 2 -name macos -type d -print -quit); then + echo "found QTDIR at $QTDIR" + else + echo "could not find QTDIR!" + exit 2 + fi + # the qtdir is located at Qt/[qtversion]/macos + # we use find to get the first subfolder with the name "macos" + # this works independent of the qt version as there should be only one version installed on the runner at a time + export QTDIR if [[ $TARGET_MACOS_VERSION ]]; then # CMAKE_OSX_DEPLOYMENT_TARGET is a vanilla cmake flag needed to compile to target macOS version diff --git a/.ci/resolve_latest_aqt_qt_version.sh b/.ci/resolve_latest_aqt_qt_version.sh new file mode 100755 index 000000000..154c15321 --- /dev/null +++ b/.ci/resolve_latest_aqt_qt_version.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script is used to resolve the latest patch version of Qt using aqtinstall. +# It interprets wildcards to get the latest patch version. E.g. "6.6.*" -> "6.6.3". + +# This script is meant to be used by the ci enironment. +# It uses the runner's GITHUB_OUTPUT env variable. + +# Usage example: .ci/resolve_latest_aqt_qt_version.sh "6.6.*" + +qt_spec=$1 +if [[ ! $qt_spec ]]; then + echo "usage: $0 [version]" + exit 2 +fi + +# If version is already specific (no wildcard), use it as-is +if [[ $qt_spec != *"*" ]]; then + echo "version $qt_spec is already resolved" + echo "version=$qt_spec" >> "$GITHUB_OUTPUT" + exit 0 +fi + +if ! hash aqt; then + echo "aqt could not be found, has aqtinstall been installed?" + exit 2 +fi + +# Resolve latest patch +if ! qt_resolved=$(aqt list-qt mac desktop --spec "$qt_spec" --latest-version); then + exit 1 +fi + +echo "resolved $qt_spec to $qt_resolved" +if [[ ! $qt_resolved ]]; then + echo "Error: Could not resolve Qt version for $qt_spec" + exit 1 +fi + +echo "version=$qt_resolved" >> "$GITHUB_OUTPUT" diff --git a/.ci/thin_macos_qtlib.sh b/.ci/thin_macos_qtlib.sh new file mode 100755 index 000000000..6c3cc6720 --- /dev/null +++ b/.ci/thin_macos_qtlib.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# The macos binaries from aqt are fat (universal), so we thin them to the target architecture to reduce the size of +# the packages and caches using lipo. + +# This script is meant to be used by the ci enironment on macos runners only. +# It uses the runner's GITHUB_WORKSPACE env variable. +arch=$(uname -m) +nproc=$(sysctl -n hw.ncpu) + +function thin() { + local libfile=$1 + if [[ $(file -b --mime-type "$libfile") == application/x-mach-binary* ]]; then + echo "Processing $libfile" + lipo "$libfile" -thin "$arch" -output "$libfile" + fi + return 0 +} +export -f thin # export to allow use in xargs +export arch +set -eo pipefail + +echo "::group::Thinning Qt libraries to $arch using $nproc cores" +find "$GITHUB_WORKSPACE/Qt" -type f -print0 | xargs -0 -n1 -P"$nproc" -I{} bash -c "thin '{}'" +echo "::endgroup::" diff --git a/.github/workflows/desktop-build.yml b/.github/workflows/desktop-build.yml index 9f4ee4984..5b954d021 100644 --- a/.github/workflows/desktop-build.yml +++ b/.github/workflows/desktop-build.yml @@ -262,7 +262,6 @@ jobs: qt_version: 6.6.* qt_arch: clang_64 qt_modules: qtimageformats qtmultimedia qtwebsockets - cache_qt: false # qt caches take too much space for macOS (1.1Gi) cmake_generator: Ninja use_ccache: 1 @@ -278,7 +277,6 @@ jobs: qt_version: 6.6.* qt_arch: clang_64 qt_modules: qtimageformats qtmultimedia qtwebsockets - cache_qt: false cmake_generator: Ninja use_ccache: 1 @@ -294,7 +292,6 @@ jobs: qt_version: 6.6.* qt_arch: clang_64 qt_modules: qtimageformats qtmultimedia qtwebsockets - cache_qt: false cmake_generator: Ninja use_ccache: 1 @@ -307,7 +304,6 @@ jobs: qt_version: 6.6.* qt_arch: clang_64 qt_modules: qtimageformats qtmultimedia qtwebsockets - cache_qt: false cmake_generator: Ninja use_ccache: 1 @@ -320,7 +316,6 @@ jobs: artifact_name: Windows7-installer qt_version: 5.15.* qt_arch: win64_msvc2019_64 - cache_qt: true cmake_generator: "Visual Studio 17 2022" cmake_generator_platform: x64 @@ -334,7 +329,6 @@ jobs: qt_version: 6.6.* qt_arch: win64_msvc2019_64 qt_modules: qtimageformats qtmultimedia qtwebsockets - cache_qt: true cmake_generator: "Visual Studio 17 2022" cmake_generator_platform: x64 @@ -375,13 +369,57 @@ jobs: key: ccache-${{matrix.runner}}-${{matrix.soc}}-${{matrix.type}}-${{env.BRANCH_NAME}} restore-keys: ccache-${{matrix.runner}}-${{matrix.soc}}-${{matrix.type}}- - - name: Install Qt ${{matrix.qt_version}} + - name: Install aqtinstall + if: matrix.os == 'macOS' + run: pipx install aqtinstall + + # Checking if there's a newer, uncached version of Qt available to install via aqtinstall + - name: Resolve latest Qt patch version + if: matrix.os == 'macOS' + id: resolve_qt_version + shell: bash + # Ouputs the version of Qt to install via aqtinstall + run: .ci/resolve_latest_aqt_qt_version.sh "${{matrix.qt_version}}" + + - name: Restore thin Qt ${{ steps.resolve_qt_version.outputs.version }} libraries (${{ matrix.soc }} macOS) + if: matrix.os == 'macOS' + id: restore_qt + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/Qt + key: thin-qt-macos-${{ matrix.soc }}-${{ steps.resolve_qt_version.outputs.version }} + + # Using jurplel/install-qt-action to install Qt without using brew + # qt build using vcpkg either just fails or takes too long to build + - name: Install fat Qt ${{ steps.resolve_qt_version.outputs.version }} (${{ matrix.soc }} macOS) + if: matrix.os == 'macOS' && steps.restore_qt.outputs.cache-hit != 'true' + uses: jurplel/install-qt-action@v4 + with: + cache: false + version: ${{ steps.resolve_qt_version.outputs.version }} + arch: ${{matrix.qt_arch}} + modules: ${{matrix.qt_modules}} + dir: ${{github.workspace}} + + - name: Thin Qt libraries (${{ matrix.soc }} macOS) + if: matrix.os == 'macOS' && steps.restore_qt.outputs.cache-hit != 'true' + run: .ci/thin_macos_qtlib.sh + + - name: Cache thin Qt libraries (${{ matrix.soc }} macOS) + if: matrix.os == 'macOS' && steps.restore_qt.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ github.workspace }}/Qt + key: thin-qt-macos-${{ matrix.soc }}-${{ steps.resolve_qt_version.outputs.version }} + + - name: Install Qt ${{matrix.qt_version}} (Windows) + if: matrix.os == 'Windows' uses: jurplel/install-qt-action@v4 with: version: ${{matrix.qt_version}} arch: ${{matrix.qt_arch}} modules: ${{matrix.qt_modules}} - cache: ${{matrix.cache_qt}} + cache: true - name: Setup vcpkg cache id: vcpkg-cache