new: Add cache base warmup strategy. (#16)

This commit is contained in:
Miles Johnson 2023-11-17 10:27:56 -08:00 committed by GitHub
parent c15c42022b
commit c91b4202a2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 177 additions and 86 deletions

View file

@ -9,20 +9,20 @@ jobs:
name: 'CI' name: 'CI'
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
- uses: actions/setup-node@v3 - uses: actions/setup-node@v3
- run: npm install -g pnpm - run: npm install -g pnpm
- run: pnpm install - run: pnpm install
- run: pnpm run check - run: pnpm run check
action: action-default:
name: 'Action' name: 'Action - Default'
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
strategy: strategy:
matrix: matrix:
os: [ubuntu-latest, macos-latest, windows-latest] os: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false fail-fast: false
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
- uses: actions/setup-node@v3 - uses: actions/setup-node@v3
- run: npm install -g pnpm - run: npm install -g pnpm
- run: pnpm install - run: pnpm install
@ -30,3 +30,21 @@ jobs:
- uses: ./ # self - uses: ./ # self
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
action-warmup:
name: 'Action - Cache warmup'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v3
- run: npm install -g pnpm
- run: pnpm install
- run: pnpm run build
- uses: ./ # self
with:
cache-base: master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -6,7 +6,7 @@ jobs:
build: build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v4
with: with:
ref: ${{ github.event.release.tag_name }} ref: ${{ github.event.release.tag_name }}
- uses: actions/setup-node@v3 - uses: actions/setup-node@v3

View file

@ -1,3 +1,9 @@
# 1.1.0
- Added a `cache-base` input. When provided, will only save cache on this branch/ref, but will
restore cache on all branches/refs.
- Updated dependencies.
# 1.0.3 # 1.0.3
- Include `GITHUB_WORKFLOW` in cache key. - Include `GITHUB_WORKFLOW` in cache key.

View file

@ -22,6 +22,8 @@ optional.
- `bins` - Comma-separated list of global binaries to install into Cargo. - `bins` - Comma-separated list of global binaries to install into Cargo.
- `cache` - Toggle caching of directories. Defaults to `true`. - `cache` - Toggle caching of directories. Defaults to `true`.
- `cache-base` - Base branch/ref to save a warmup cache on. Other branches/refs will restore from
this base.
- `cache-target` - Name of the target profile to cache. Defaults to `debug`. - `cache-target` - Name of the target profile to cache. Defaults to `debug`.
- `channel` - Toolchain specification/channel to explicitly install. - `channel` - Toolchain specification/channel to explicitly install.
- `components` - Comma-separated list of additional components to install. - `components` - Comma-separated list of additional components to install.
@ -135,6 +137,23 @@ The following optimizations and considerations are taken into account when cachi
> The following sources are hashed for the generated cache key: `$GITHUB_JOB`, `Cargo.lock`, Rust > The following sources are hashed for the generated cache key: `$GITHUB_JOB`, `Cargo.lock`, Rust
> version, Rust commit hash, and OS. > version, Rust commit hash, and OS.
### Warmup strategy
Another strategy that we support is called a warmup cache, where a base branch/ref is used to
generate and save the cache (like master), and all other branches/refs will _only_ restore this
cache (and not save).
This can be enabled with the `cache-base` input, which requires a branch/ref name. This input also
supports regex.
```yaml
- uses: moonrepo/setup-rust@v1
with:
cache-base: master
# With regex
cache-base: (master|main|develop)
```
## Compared to ## Compared to
### `actions-rs/*` ### `actions-rs/*`

View file

@ -9,6 +9,9 @@ inputs:
cache: cache:
description: 'Toggle caching of ~/.cargo/registry and /target/<cache-target> directories.' description: 'Toggle caching of ~/.cargo/registry and /target/<cache-target> directories.'
default: true default: true
cache-base:
description:
'Base branch/ref to save a warmup cache on. Other branches/refs will restore from this base.'
cache-target: cache-target:
description: 'Name of the target profile to cache.' description: 'Name of the target profile to cache.'
default: 'debug' default: 'debug'

View file

@ -5,7 +5,8 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec'; import * as exec from '@actions/exec';
import * as io from '@actions/io'; import * as io from '@actions/io';
import * as tc from '@actions/tool-cache'; import * as tc from '@actions/tool-cache';
import { CARGO_HOME, installBins, restoreCache } from './src/cargo'; import { CARGO_HOME } from './src/cache';
import { installBins, restoreCache } from './src/cargo';
import { installToolchain } from './src/rust'; import { installToolchain } from './src/rust';
export async function installRustup() { export async function installRustup() {

View file

@ -26,11 +26,11 @@
"detect-libc": "^2.0.2" "detect-libc": "^2.0.2"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^20.9.0", "@types/node": "^20.9.1",
"@vercel/ncc": "^0.38.1", "@vercel/ncc": "^0.38.1",
"eslint": "^8.53.0", "eslint": "^8.53.0",
"eslint-config-moon": "^2.0.11", "eslint-config-moon": "^2.0.11",
"prettier": "^3.0.3", "prettier": "^3.1.0",
"prettier-config-moon": "^1.1.2", "prettier-config-moon": "^1.1.2",
"tsconfig-moon": "^1.3.0", "tsconfig-moon": "^1.3.0",
"typescript": "^5.2.2" "typescript": "^5.2.2"

20
pnpm-lock.yaml generated
View file

@ -32,8 +32,8 @@ dependencies:
devDependencies: devDependencies:
'@types/node': '@types/node':
specifier: ^20.9.0 specifier: ^20.9.1
version: 20.9.0 version: 20.9.1
'@vercel/ncc': '@vercel/ncc':
specifier: ^0.38.1 specifier: ^0.38.1
version: 0.38.1 version: 0.38.1
@ -44,8 +44,8 @@ devDependencies:
specifier: ^2.0.11 specifier: ^2.0.11
version: 2.0.11(eslint@8.53.0)(typescript@5.2.2) version: 2.0.11(eslint@8.53.0)(typescript@5.2.2)
prettier: prettier:
specifier: ^3.0.3 specifier: ^3.1.0
version: 3.0.3 version: 3.1.0
prettier-config-moon: prettier-config-moon:
specifier: ^1.1.2 specifier: ^1.1.2
version: 1.1.2 version: 1.1.2
@ -383,12 +383,12 @@ packages:
/@types/node-fetch@2.6.9: /@types/node-fetch@2.6.9:
resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==} resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==}
dependencies: dependencies:
'@types/node': 20.9.0 '@types/node': 20.9.1
form-data: 4.0.0 form-data: 4.0.0
dev: false dev: false
/@types/node@20.9.0: /@types/node@20.9.1:
resolution: {integrity: sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==} resolution: {integrity: sha512-HhmzZh5LSJNS5O8jQKpJ/3ZcrrlG6L70hpGqMIAoM9YVD0YBRNWYsfwcXq8VnSjlNpCpgLzMXdiPo+dxcvSmiA==}
dependencies: dependencies:
undici-types: 5.26.5 undici-types: 5.26.5
@ -403,7 +403,7 @@ packages:
/@types/tunnel@0.0.3: /@types/tunnel@0.0.3:
resolution: {integrity: sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==} resolution: {integrity: sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==}
dependencies: dependencies:
'@types/node': 20.9.0 '@types/node': 20.9.1
dev: false dev: false
/@typescript-eslint/eslint-plugin@6.10.0(@typescript-eslint/parser@6.10.0)(eslint@8.53.0)(typescript@5.2.2): /@typescript-eslint/eslint-plugin@6.10.0(@typescript-eslint/parser@6.10.0)(eslint@8.53.0)(typescript@5.2.2):
@ -2486,8 +2486,8 @@ packages:
resolution: {integrity: sha512-zShTjMXH4GlracR3jllxsYJGlYAh5w75TZUuFW6YG75unimMMCNfkWY6EbI5nqr4T+xhb81rDMq2m5YxopBfiQ==} resolution: {integrity: sha512-zShTjMXH4GlracR3jllxsYJGlYAh5w75TZUuFW6YG75unimMMCNfkWY6EbI5nqr4T+xhb81rDMq2m5YxopBfiQ==}
dev: true dev: true
/prettier@3.0.3: /prettier@3.1.0:
resolution: {integrity: sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==} resolution: {integrity: sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw==}
engines: {node: '>=14'} engines: {node: '>=14'}
hasBin: true hasBin: true
dev: true dev: true

View file

@ -3,7 +3,14 @@ import { saveCache } from './src/cargo';
async function run() { async function run() {
try { try {
await saveCache(); const base = core.getInput('cache-base');
// Only save the cache for the following 2 scenarios:
// - If not using the base warmup strategy.
// - If using the base warmup strategy, and the current ref matches.
if (!base || (base && !!(process.env.GITHUB_REF_NAME ?? '').match(base))) {
await saveCache();
}
} catch (error: unknown) { } catch (error: unknown) {
core.setFailed((error as Error).message); core.setFailed((error as Error).message);
} }

92
src/cache.ts Normal file
View file

@ -0,0 +1,92 @@
import crypto from 'node:crypto';
import os from 'node:os';
import path from 'node:path';
import * as cache from '@actions/cache';
import * as core from '@actions/core';
import * as glob from '@actions/glob';
import { RUST_HASH, RUST_VERSION } from './rust';
export const CARGO_HOME = process.env.CARGO_HOME ?? path.join(os.homedir(), '.cargo');
export const WORKSPACE_ROOT = process.env.GITHUB_WORKSPACE ?? process.cwd();
export function isCacheEnabled(): boolean {
return core.getBooleanInput('cache') && cache.isFeatureAvailable();
}
export function getCacheTarget(): string {
return core.getInput('cache-target') || 'debug';
}
export function getCachePaths(): string[] {
return [
// ~/.cargo/registry
path.join(CARGO_HOME, 'registry'),
// /workspace/target/debug
path.join(WORKSPACE_ROOT, 'target', getCacheTarget()),
];
}
export function getCachePrefixes(): string[] {
return [`setup-rustcargo-v1-${process.platform}`, 'setup-rustcargo-v1'];
}
export async function getPrimaryCacheKey() {
const hasher = crypto.createHash('sha1');
core.info('Generating cache key');
core.debug(`Hashing Rust version = ${RUST_VERSION}`);
hasher.update(RUST_VERSION);
core.debug(`Hashing Rust commit hash = ${RUST_HASH}`);
hasher.update(RUST_HASH);
const cacheTarget = getCacheTarget();
core.debug(`Hashing target profile = ${cacheTarget}`);
hasher.update(cacheTarget);
// When warming up, loosen the cache key to allow for more cache hits
if (core.getInput('cache-base')) {
core.debug('Using warmup strategy, not hashing Cargo.lock, GITHUB_WORKFLOW, or GITHUB_JOB');
hasher.update('warmup');
const baseRef = process.env.GITHUB_BASE_REF ?? '';
if (
baseRef === 'master' ||
baseRef === 'main' ||
baseRef === 'trunk' ||
baseRef.startsWith('develop') ||
baseRef.startsWith('release')
) {
core.debug(`Hashing GITHUB_BASE_REF = ${baseRef}`);
hasher.update(baseRef);
}
}
// Otherwise, these add far too much granularity to the cache key
else {
const lockHash = await glob.hashFiles('Cargo.lock');
core.debug(`Hashing Cargo.lock = ${lockHash}`);
hasher.update(lockHash);
const workflow = process.env.GITHUB_WORKFLOW;
if (workflow) {
core.debug(`Hashing GITHUB_WORKFLOW = ${workflow}`);
hasher.update(workflow);
}
const job = process.env.GITHUB_JOB;
if (job) {
core.debug(`Hashing GITHUB_JOB = ${job}`);
hasher.update(job);
}
}
return `${getCachePrefixes()[0]}-${hasher.digest('hex')}`;
}

View file

@ -1,6 +1,4 @@
import crypto from 'node:crypto';
import fs from 'node:fs'; import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path'; import path from 'node:path';
import { family } from 'detect-libc'; import { family } from 'detect-libc';
import * as cache from '@actions/cache'; import * as cache from '@actions/cache';
@ -8,14 +6,16 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec'; import * as exec from '@actions/exec';
import * as glob from '@actions/glob'; import * as glob from '@actions/glob';
import * as tc from '@actions/tool-cache'; import * as tc from '@actions/tool-cache';
import {
CARGO_HOME,
getCachePaths,
getCachePrefixes,
getCacheTarget,
getPrimaryCacheKey,
isCacheEnabled,
WORKSPACE_ROOT,
} from './cache';
import { rmrf } from './fs'; import { rmrf } from './fs';
import { RUST_HASH, RUST_VERSION } from './rust';
export const CARGO_HOME = process.env.CARGO_HOME ?? path.join(os.homedir(), '.cargo');
export const WORKSPACE_ROOT = process.env.GITHUB_WORKSPACE ?? process.cwd();
export const CACHE_ENABLED = core.getBooleanInput('cache') && cache.isFeatureAvailable();
export async function downloadAndInstallBinstall(binDir: string) { export async function downloadAndInstallBinstall(binDir: string) {
core.info('cargo-binstall does not exist, attempting to install'); core.info('cargo-binstall does not exist, attempting to install');
@ -79,7 +79,7 @@ export async function installBins() {
.map((bin) => bin.trim()) .map((bin) => bin.trim())
.filter(Boolean); .filter(Boolean);
if (CACHE_ENABLED) { if (isCacheEnabled()) {
bins.push('cargo-cache'); bins.push('cargo-cache');
} }
@ -98,61 +98,6 @@ export async function installBins() {
await exec.exec('cargo', ['binstall', '--no-confirm', '--log-level', 'info', ...bins]); await exec.exec('cargo', ['binstall', '--no-confirm', '--log-level', 'info', ...bins]);
} }
export function getCacheTarget(): string {
return core.getInput('cache-target') || 'debug';
}
export function getCachePaths(): string[] {
return [
// ~/.cargo/registry
path.join(CARGO_HOME, 'registry'),
// /workspace/target/debug
path.join(WORKSPACE_ROOT, 'target', getCacheTarget()),
];
}
export function getCachePrefixes(): string[] {
return [`setup-rustcargo-v1-${process.platform}`, 'setup-rustcargo-v1'];
}
export async function getPrimaryCacheKey() {
const hasher = crypto.createHash('sha1');
core.info('Generating cache key');
core.debug(`Hashing Rust version = ${RUST_VERSION}`);
hasher.update(RUST_VERSION);
core.debug(`Hashing Rust commit hash = ${RUST_HASH}`);
hasher.update(RUST_HASH);
const lockHash = await glob.hashFiles('Cargo.lock');
core.debug(`Hashing Cargo.lock = ${lockHash}`);
hasher.update(lockHash);
const cacheTarget = getCacheTarget();
core.debug(`Hashing target profile = ${cacheTarget}`);
hasher.update(cacheTarget);
const workflow = process.env.GITHUB_WORKFLOW;
if (workflow) {
core.debug(`Hashing GITHUB_WORKFLOW = ${workflow}`);
hasher.update(workflow);
}
const job = process.env.GITHUB_JOB;
if (job) {
core.debug(`Hashing GITHUB_JOB = ${job}`);
hasher.update(job);
}
return `${getCachePrefixes()[0]}-${hasher.digest('hex')}`;
}
export async function cleanCargoRegistry() { export async function cleanCargoRegistry() {
core.info('Cleaning ~/.cargo before saving'); core.info('Cleaning ~/.cargo before saving');
@ -208,7 +153,7 @@ export async function cleanTargetProfile() {
} }
export async function saveCache() { export async function saveCache() {
if (!CACHE_ENABLED) { if (!isCacheEnabled()) {
return; return;
} }
@ -229,7 +174,7 @@ export async function saveCache() {
} }
export async function restoreCache() { export async function restoreCache() {
if (!CACHE_ENABLED) { if (!isCacheEnabled()) {
return; return;
} }