diff --git a/.github/actions/ccache-clear/action.yml b/.github/actions/ccache-clear/action.yml index 2420045edf..fc5da4f6ed 100644 --- a/.github/actions/ccache-clear/action.yml +++ b/.github/actions/ccache-clear/action.yml @@ -21,68 +21,30 @@ inputs: runs: using: "composite" steps: + - name: Install GitHub CLI if missing + shell: bash + run: | + # e.g. in container jobs, where it is not preinstalled + if ! command -v gh >/dev/null 2>&1; then + echo "GitHub CLI not found, installing..." + if ! command -v curl >/dev/null 2>&1; then + apt-get update >/dev/null 2>&1 || true + apt-get install -y curl >/dev/null 2>&1 || true + fi + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg >/dev/null + chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list + apt-get update >/dev/null 2>&1 || true + apt-get install -y gh || { echo "Failed to install GitHub CLI (gh)" >&2; exit 1; } + fi + command -v gh >/dev/null 2>&1 || { echo "GitHub CLI (gh) is required but could not be installed" >&2; exit 1; } + - name: Clear caches shell: bash - env: - CLEAR_KEY: ${{ inputs.key }} - CLEAR_OLDER: ${{ inputs.older }} - CLEAR_MIN: ${{ inputs.min }} - CLEAR_DRY_RUN: ${{ inputs.dry-run }} run: | - # Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds - to_seconds() { - local val="$1" - [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; } - local num="${val%?}" unit="${val: -1}" mult - [[ "$num" =~ ^[0-9]+$ ]] || return 1 - case "$unit" in - s) mult=1 ;; - m) mult=60 ;; - h) mult=3600 ;; - d) mult=86400 ;; - *) return 1 ;; - esac - echo $((num * mult)) - } - - [[ "$CLEAR_MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $CLEAR_MIN" >&2; exit 1; } - [[ "$CLEAR_DRY_RUN" =~ ^(true|false)$ ]] || { echo "Invalid dry-run value: $CLEAR_DRY_RUN" >&2; exit 1; } - - CACHES=$(gh cache list --key "ccache-$CLEAR_KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' 2>/dev/null | LC_ALL=C sort) - if [ -z "$CACHES" ]; then - echo "No caches found with key prefix: $CLEAR_KEY" - exit 0 - fi - - TOTAL=$(( $(wc -l <<< "$CACHES") )) - - echo "Found $TOTAL cache(s) with key prefix: $CLEAR_KEY (oldest first):" - while IFS=$'\t' read -r CREATED ID KEY; do - printf ' %s %s %s\n' "$CREATED" "$ID" "$KEY" - done <<< "$CACHES" - - CUTOFF="" - if [ -n "$CLEAR_OLDER" ]; then - OLDER_SECONDS=$(to_seconds "$CLEAR_OLDER") || { echo "Invalid older value: $CLEAR_OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; } - CUTOFF=$(( $(date +%s) - OLDER_SECONDS )) - fi - - # Caches are sorted oldest first - DELETED=0 - while IFS=$'\t' read -r CREATED ID KEY; do - if [ -n "$CUTOFF" ] && [ "$(date -d "$CREATED" +%s)" -ge "$CUTOFF" ]; then - echo "Rest are not older than $CLEAR_OLDER, stopping" - break - fi - if [ $((TOTAL - DELETED - 1)) -lt "$CLEAR_MIN" ]; then - echo "Keeping at least $CLEAR_MIN cache(s), stopping" - break - fi - if [ "$CLEAR_DRY_RUN" = "true" ]; then - echo "Would delete cache: $ID ($KEY)" - else - echo "Deleting cache: $ID ($KEY)" - gh cache delete "$ID" - fi - DELETED=$((DELETED + 1)) - done <<< "$CACHES" + bash scripts/ccache-clear.sh \ + --key "${{ inputs.key }}" \ + --older "${{ inputs.older }}" \ + --min "${{ inputs.min }}" \ + ${{ inputs.dry-run == 'true' && '--dry-run' || '' }} diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml index 289e5144e6..9a4a691d52 100644 --- a/.github/workflows/build-apple.yml +++ b/.github/workflows/build-apple.yml @@ -73,6 +73,16 @@ jobs: cd build ctest -L main -E "test-llama-archs" --verbose --timeout 900 + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: apple-arm64 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + macos-latest-x64: runs-on: macos-15-intel @@ -109,6 +119,16 @@ jobs: cd build ctest -L main --verbose --timeout 900 + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: apple-x64 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + macos-latest-ios-xcode: runs-on: macos-latest @@ -163,14 +183,6 @@ jobs: id: checkout uses: actions/checkout@v6 - # TODO: this likely does not do anything - if yes, remove it - - name: ccache - uses: ggml-org/ccache-action@v1.2.21 - with: - key: apple-tvos - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - name: Build id: cmake_build run: | @@ -196,14 +208,6 @@ jobs: id: checkout uses: actions/checkout@v6 - # TODO: this likely does not do anything - if yes, remove it - - name: ccache - uses: ggml-org/ccache-action@v1.2.21 - with: - key: apple-visionos - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - name: Build id: cmake_build run: | @@ -234,14 +238,6 @@ jobs: id: checkout uses: actions/checkout@v6 - # TODO: this likely does not do anything - if yes, remove it - - name: ccache - uses: ggml-org/ccache-action@v1.2.21 - with: - key: apple-swift - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - - name: Download xcframework artifact uses: actions/download-artifact@v7 with: diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index f39304cabf..b62fe55d6c 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -125,7 +125,7 @@ jobs: GH_TOKEN: ${{ github.token }} with: key: cpu-${{ matrix.os }} - older: 1h + older: 5m min: 1 dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} @@ -215,3 +215,13 @@ jobs: # cd build # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1 # & $sde -future -- ctest -L main -C Release --verbose --timeout 900 + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: cpu-windows-2025-${{ matrix.build }} + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index 2528b18573..ed5164756a 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -72,6 +72,16 @@ jobs: -DGGML_CUDA_CUB_3DOT2=ON cmake --build build + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: cuda-ubuntu-24.04-cuda + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + hip: runs-on: ubuntu-22.04 container: rocm/dev-ubuntu-22.04:6.1.2 @@ -103,6 +113,16 @@ jobs: -DGGML_HIP=ON cmake --build build --config Release -j $(nproc) + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: cuda-ubuntu-22.04-hip + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + musa: runs-on: ubuntu-22.04 container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64 @@ -131,3 +151,13 @@ jobs: cmake -B build -S . \ -DGGML_MUSA=ON time cmake --build build --config Release -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: cuda-ubuntu-22.04-musa + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-opencl.yml b/.github/workflows/build-opencl.yml index 251b1f8d59..c0adc7e496 100644 --- a/.github/workflows/build-opencl.yml +++ b/.github/workflows/build-opencl.yml @@ -80,3 +80,13 @@ jobs: run: | cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: opencl-windows-2025-x64 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index ee4268f970..0316e7ad97 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -167,3 +167,13 @@ jobs: cd build ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000 + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: openvino-windows-2022 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-sycl.yml b/.github/workflows/build-sycl.yml index deb0e5479a..7beac81772 100644 --- a/.github/workflows/build-sycl.yml +++ b/.github/workflows/build-sycl.yml @@ -96,6 +96,16 @@ jobs: -DGGML_SYCL_F16=${{ matrix.fp16 }} time cmake --build build --config Release -j $(nproc) + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: sycl-ubuntu-24-${{ matrix.build }} + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + windows-latest-sycl: runs-on: windows-2022 @@ -139,3 +149,13 @@ jobs: - name: Build id: cmake_build run: examples/sycl/win-build-sycl.bat + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: sycl-windows-latest + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index 15ff4b0af3..74d1c69368 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -55,7 +55,7 @@ jobs: - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: - key: vulkan-ubuntu-24.04-arm-new + key: vulkan-ubuntu-24.04-arm variant: ccache evict-old-files: 1d save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} @@ -73,6 +73,16 @@ jobs: run: | time cmake --build build -j $(nproc) + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: vulkan-ubuntu-24.04-arm + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + ubuntu-llvmpipe: runs-on: ubuntu-24.04 @@ -128,6 +138,16 @@ jobs: # test-backend-ops is too slow on llvmpipe, skip it ctest -L main -E test-backend-ops --verbose --timeout 900 + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: vulkan-ubuntu-24.04-llvmpipe + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + windows: runs-on: windows-2025 @@ -180,3 +200,13 @@ jobs: run: | cd build ctest -L main -C Release --verbose --timeout 900 + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: cpu-windows-2025-x64-vulkan + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index aa7ae887dc..2e4680f385 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -88,3 +88,13 @@ jobs: -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc) + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: webgpu-ubuntu-24.04-arm-wasm + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build-webgpu.yml b/.github/workflows/build-webgpu.yml index ed73c185aa..b357851aa2 100644 --- a/.github/workflows/build-webgpu.yml +++ b/.github/workflows/build-webgpu.yml @@ -101,6 +101,16 @@ jobs: cd build ctest -L main --verbose --timeout 900 + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: webgpu-macos-latest + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + ubuntu: runs-on: ubuntu-24.04 @@ -153,3 +163,13 @@ jobs: # This is using llvmpipe and runs slower than other backends # test-backend-ops is too slow on llvmpipe, skip it ctest -L main -E test-backend-ops --verbose --timeout 900 + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: webgpu-ubuntu-24.04 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml index 5d23f01cf8..ecc4615a1a 100644 --- a/.github/workflows/hip-quality-check.yml +++ b/.github/workflows/hip-quality-check.yml @@ -84,3 +84,13 @@ jobs: cd build make -j $(nproc) 2>&1 | tee metrics.log | grep -v 'Rpass-analysis=kernel-resource-usage\|remark:\|^$' python3 ../scripts/hip/gcn-cdna-vgpr-check.py metrics.log + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: hip-quality-check-ubuntu-22.04 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 9fb4b4ba10..530ace7cda 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -128,6 +128,16 @@ jobs: export LLAMA_ARG_BACKEND_SAMPLING=1 SLOW_TESTS=1 ./tests.sh + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: server-ubuntu-24.04-arm + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + windows: runs-on: windows-2025 @@ -181,3 +191,13 @@ jobs: cd tools/server/tests export SLOW_TESTS="1" ./tests.sh + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + env: + GH_TOKEN: ${{ github.token }} + with: + key: server-windows-2025-x64 + older: 5m + min: 1 + dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} diff --git a/scripts/ccache-clear.sh b/scripts/ccache-clear.sh new file mode 100755 index 0000000000..27fda33156 --- /dev/null +++ b/scripts/ccache-clear.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Delete GitHub Actions caches matching a key prefix, oldest first. +# +# Usage: ccache-clear.sh --key KEY [--older DURATION] [--min N] [--dry-run] +# --key: cache key prefix to match and delete (without the ccache- prefix) +# --older: only delete caches created more than DURATION ago (e.g. 5m, 1h, 1d); +# by default all matching caches are deleted +# --min: stop deleting if fewer than N caches would remain (default: 0) +# --dry-run: only print the caches that would be deleted, without deleting them +# +# Env (when running in GitHub Actions): +# GH_TOKEN: token for the gh CLI +# GITHUB_REPOSITORY: owner/repo of the caches to manage +set -euo pipefail + +KEY="" +OLDER="" +MIN=0 +DRY_RUN=false +while [[ $# -gt 0 ]]; do + case "$1" in + --key) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; KEY="$2"; shift 2 ;; + --older) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; OLDER="$2"; shift 2 ;; + --min) [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 1; }; MIN="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +command -v gh >/dev/null 2>&1 || { echo "Error: GitHub CLI (gh) is required" >&2; exit 1; } +[[ -n "${GITHUB_REPOSITORY:-}" ]] || { echo "Error: GITHUB_REPOSITORY not set" >&2; exit 1; } +[[ -n "$KEY" ]] || { echo "Error: --key is required" >&2; exit 1; } +[[ "$MIN" =~ ^[0-9]+$ ]] || { echo "Invalid min value: $MIN" >&2; exit 1; } + +# Convert a duration (e.g. 90m, 1h, 1d, plain seconds) to seconds +to_seconds() { + local val="$1" + [[ "$val" =~ ^[0-9]+$ ]] && { echo "$val"; return 0; } + local num="${val%?}" unit="${val: -1}" mult + [[ "$num" =~ ^[0-9]+$ ]] || return 1 + case "$unit" in + s) mult=1 ;; + m) mult=60 ;; + h) mult=3600 ;; + d) mult=86400 ;; + *) return 1 ;; + esac + echo $((num * mult)) +} + +# Convert an ISO-8601 UTC timestamp (e.g. 2026-08-23T16:51:23.313693Z) to epoch seconds +to_epoch() { + local val="$1" out + # GNU date (e.g. Linux) + if out=$(date -d "$val" +%s 2>/dev/null) && [[ "$out" =~ ^[0-9]+$ ]]; then + echo "$out" + return 0 + fi + # BSD date (e.g. macOS); fractional seconds are not needed, TZ forces UTC + out=$(TZ=UTC date -j -f "%Y-%m-%dT%H:%M:%S" "${val:0:19}" +%s 2>/dev/null) || return 1 + [[ "$out" =~ ^[0-9]+$ ]] || return 1 + echo "$out" +} + +CACHES=$(gh cache list --repo "$GITHUB_REPOSITORY" --key "ccache-$KEY" --json id,key,createdAt --jq '.[] | [.createdAt, .id, .key] | @tsv' | LC_ALL=C sort) +if [[ -z "$CACHES" ]]; then + echo "No caches found with key prefix: $KEY" + exit 0 +fi + +TOTAL=$(( $(wc -l <<< "$CACHES") )) + +echo "Found $TOTAL cache(s) with key prefix: $KEY (oldest first):" +while IFS=$'\t' read -r CREATED ID CACHE_KEY; do + printf ' %s %s %s\n' "$CREATED" "$ID" "$CACHE_KEY" +done <<< "$CACHES" + +CUTOFF="" +if [[ -n "$OLDER" ]]; then + OLDER_SECONDS=$(to_seconds "$OLDER") || { echo "Invalid older value: $OLDER (expected e.g. 90m, 1h, 1d)" >&2; exit 1; } + CUTOFF=$(( $(date +%s) - OLDER_SECONDS )) +fi + +# Caches are sorted oldest first +DELETED=0 +while IFS=$'\t' read -r CREATED ID CACHE_KEY; do + if [[ -n "$CUTOFF" ]]; then + CREATED_SECONDS=$(to_epoch "$CREATED") || { echo "Failed to parse date: $CREATED" >&2; exit 1; } + if [[ "$CREATED_SECONDS" -ge "$CUTOFF" ]]; then + echo "Rest are not older than $OLDER, stopping" + break + fi + fi + if (( TOTAL - DELETED - 1 < MIN )); then + echo "Keeping at least $MIN cache(s), stopping" + break + fi + if [[ "$DRY_RUN" == "true" ]]; then + echo "Would delete cache: $ID ($CACHE_KEY)" + else + echo "Deleting cache: $ID ($CACHE_KEY)" + gh cache delete --repo "$GITHUB_REPOSITORY" "$ID" + fi + DELETED=$((DELETED + 1)) +done <<< "$CACHES"