new: Add cache base warmup strategy. (#16)

This commit is contained in:
Miles Johnson 2023-11-17 10:27:56 -08:00 committed by GitHub
parent c15c42022b
commit c91b4202a2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 177 additions and 86 deletions

View file

@ -9,20 +9,20 @@ jobs:
name: 'CI'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-node@v3
- run: npm install -g pnpm
- run: pnpm install
- run: pnpm run check
action:
name: 'Action'
action-default:
name: 'Action - Default'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-node@v3
- run: npm install -g pnpm
- run: pnpm install
@ -30,3 +30,21 @@ jobs:
- uses: ./ # self
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
action-warmup:
name: 'Action - Cache warmup'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
fail-fast: false
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v3
- run: npm install -g pnpm
- run: pnpm install
- run: pnpm run build
- uses: ./ # self
with:
cache-base: master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -6,7 +6,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
ref: ${{ github.event.release.tag_name }}
- uses: actions/setup-node@v3

View file

@ -1,3 +1,9 @@
# 1.1.0
- Added a `cache-base` input. When provided, will only save cache on this branch/ref, but will
restore cache on all branches/refs.
- Updated dependencies.
# 1.0.3
- Include `GITHUB_WORKFLOW` in cache key.

View file

@ -22,6 +22,8 @@ optional.
- `bins` - Comma-separated list of global binaries to install into Cargo.
- `cache` - Toggle caching of directories. Defaults to `true`.
- `cache-base` - Base branch/ref to save a warmup cache on. Other branches/refs will restore from
this base.
- `cache-target` - Name of the target profile to cache. Defaults to `debug`.
- `channel` - Toolchain specification/channel to explicitly install.
- `components` - Comma-separated list of additional components to install.
@ -135,6 +137,23 @@ The following optimizations and considerations are taken into account when cachi
> The following sources are hashed for the generated cache key: `$GITHUB_JOB`, `Cargo.lock`, Rust
> version, Rust commit hash, and OS.
### Warmup strategy
Another strategy that we support is called a warmup cache, where a base branch/ref is used to
generate and save the cache (like master), and all other branches/refs will _only_ restore this
cache (and not save).
This can be enabled with the `cache-base` input, which requires a branch/ref name. This input also
supports regex.
```yaml
- uses: moonrepo/setup-rust@v1
with:
cache-base: master
# With regex
cache-base: (master|main|develop)
```
## Compared to
### `actions-rs/*`

View file

@ -9,6 +9,9 @@ inputs:
cache:
description: 'Toggle caching of ~/.cargo/registry and /target/<cache-target> directories.'
default: true
cache-base:
description:
'Base branch/ref to save a warmup cache on. Other branches/refs will restore from this base.'
cache-target:
description: 'Name of the target profile to cache.'
default: 'debug'

View file

@ -5,7 +5,8 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as io from '@actions/io';
import * as tc from '@actions/tool-cache';
import { CARGO_HOME, installBins, restoreCache } from './src/cargo';
import { CARGO_HOME } from './src/cache';
import { installBins, restoreCache } from './src/cargo';
import { installToolchain } from './src/rust';
export async function installRustup() {

View file

@ -26,11 +26,11 @@
"detect-libc": "^2.0.2"
},
"devDependencies": {
"@types/node": "^20.9.0",
"@types/node": "^20.9.1",
"@vercel/ncc": "^0.38.1",
"eslint": "^8.53.0",
"eslint-config-moon": "^2.0.11",
"prettier": "^3.0.3",
"prettier": "^3.1.0",
"prettier-config-moon": "^1.1.2",
"tsconfig-moon": "^1.3.0",
"typescript": "^5.2.2"

20
pnpm-lock.yaml generated
View file

@ -32,8 +32,8 @@ dependencies:
devDependencies:
'@types/node':
specifier: ^20.9.0
version: 20.9.0
specifier: ^20.9.1
version: 20.9.1
'@vercel/ncc':
specifier: ^0.38.1
version: 0.38.1
@ -44,8 +44,8 @@ devDependencies:
specifier: ^2.0.11
version: 2.0.11(eslint@8.53.0)(typescript@5.2.2)
prettier:
specifier: ^3.0.3
version: 3.0.3
specifier: ^3.1.0
version: 3.1.0
prettier-config-moon:
specifier: ^1.1.2
version: 1.1.2
@ -383,12 +383,12 @@ packages:
/@types/node-fetch@2.6.9:
resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==}
dependencies:
'@types/node': 20.9.0
'@types/node': 20.9.1
form-data: 4.0.0
dev: false
/@types/node@20.9.0:
resolution: {integrity: sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==}
/@types/node@20.9.1:
resolution: {integrity: sha512-HhmzZh5LSJNS5O8jQKpJ/3ZcrrlG6L70hpGqMIAoM9YVD0YBRNWYsfwcXq8VnSjlNpCpgLzMXdiPo+dxcvSmiA==}
dependencies:
undici-types: 5.26.5
@ -403,7 +403,7 @@ packages:
/@types/tunnel@0.0.3:
resolution: {integrity: sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==}
dependencies:
'@types/node': 20.9.0
'@types/node': 20.9.1
dev: false
/@typescript-eslint/eslint-plugin@6.10.0(@typescript-eslint/parser@6.10.0)(eslint@8.53.0)(typescript@5.2.2):
@ -2486,8 +2486,8 @@ packages:
resolution: {integrity: sha512-zShTjMXH4GlracR3jllxsYJGlYAh5w75TZUuFW6YG75unimMMCNfkWY6EbI5nqr4T+xhb81rDMq2m5YxopBfiQ==}
dev: true
/prettier@3.0.3:
resolution: {integrity: sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==}
/prettier@3.1.0:
resolution: {integrity: sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw==}
engines: {node: '>=14'}
hasBin: true
dev: true

View file

@ -3,7 +3,14 @@ import { saveCache } from './src/cargo';
async function run() {
try {
await saveCache();
const base = core.getInput('cache-base');
// Only save the cache for the following 2 scenarios:
// - If not using the base warmup strategy.
// - If using the base warmup strategy, and the current ref matches.
if (!base || (base && !!(process.env.GITHUB_REF_NAME ?? '').match(base))) {
await saveCache();
}
} catch (error: unknown) {
core.setFailed((error as Error).message);
}

92
src/cache.ts Normal file
View file

@ -0,0 +1,92 @@
import crypto from 'node:crypto';
import os from 'node:os';
import path from 'node:path';
import * as cache from '@actions/cache';
import * as core from '@actions/core';
import * as glob from '@actions/glob';
import { RUST_HASH, RUST_VERSION } from './rust';
export const CARGO_HOME = process.env.CARGO_HOME ?? path.join(os.homedir(), '.cargo');
export const WORKSPACE_ROOT = process.env.GITHUB_WORKSPACE ?? process.cwd();
export function isCacheEnabled(): boolean {
return core.getBooleanInput('cache') && cache.isFeatureAvailable();
}
export function getCacheTarget(): string {
return core.getInput('cache-target') || 'debug';
}
export function getCachePaths(): string[] {
return [
// ~/.cargo/registry
path.join(CARGO_HOME, 'registry'),
// /workspace/target/debug
path.join(WORKSPACE_ROOT, 'target', getCacheTarget()),
];
}
export function getCachePrefixes(): string[] {
return [`setup-rustcargo-v1-${process.platform}`, 'setup-rustcargo-v1'];
}
export async function getPrimaryCacheKey() {
const hasher = crypto.createHash('sha1');
core.info('Generating cache key');
core.debug(`Hashing Rust version = ${RUST_VERSION}`);
hasher.update(RUST_VERSION);
core.debug(`Hashing Rust commit hash = ${RUST_HASH}`);
hasher.update(RUST_HASH);
const cacheTarget = getCacheTarget();
core.debug(`Hashing target profile = ${cacheTarget}`);
hasher.update(cacheTarget);
// When warming up, loosen the cache key to allow for more cache hits
if (core.getInput('cache-base')) {
core.debug('Using warmup strategy, not hashing Cargo.lock, GITHUB_WORKFLOW, or GITHUB_JOB');
hasher.update('warmup');
const baseRef = process.env.GITHUB_BASE_REF ?? '';
if (
baseRef === 'master' ||
baseRef === 'main' ||
baseRef === 'trunk' ||
baseRef.startsWith('develop') ||
baseRef.startsWith('release')
) {
core.debug(`Hashing GITHUB_BASE_REF = ${baseRef}`);
hasher.update(baseRef);
}
}
// Otherwise, these add far too much granularity to the cache key
else {
const lockHash = await glob.hashFiles('Cargo.lock');
core.debug(`Hashing Cargo.lock = ${lockHash}`);
hasher.update(lockHash);
const workflow = process.env.GITHUB_WORKFLOW;
if (workflow) {
core.debug(`Hashing GITHUB_WORKFLOW = ${workflow}`);
hasher.update(workflow);
}
const job = process.env.GITHUB_JOB;
if (job) {
core.debug(`Hashing GITHUB_JOB = ${job}`);
hasher.update(job);
}
}
return `${getCachePrefixes()[0]}-${hasher.digest('hex')}`;
}

View file

@ -1,6 +1,4 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { family } from 'detect-libc';
import * as cache from '@actions/cache';
@ -8,14 +6,16 @@ import * as core from '@actions/core';
import * as exec from '@actions/exec';
import * as glob from '@actions/glob';
import * as tc from '@actions/tool-cache';
import {
CARGO_HOME,
getCachePaths,
getCachePrefixes,
getCacheTarget,
getPrimaryCacheKey,
isCacheEnabled,
WORKSPACE_ROOT,
} from './cache';
import { rmrf } from './fs';
import { RUST_HASH, RUST_VERSION } from './rust';
export const CARGO_HOME = process.env.CARGO_HOME ?? path.join(os.homedir(), '.cargo');
export const WORKSPACE_ROOT = process.env.GITHUB_WORKSPACE ?? process.cwd();
export const CACHE_ENABLED = core.getBooleanInput('cache') && cache.isFeatureAvailable();
export async function downloadAndInstallBinstall(binDir: string) {
core.info('cargo-binstall does not exist, attempting to install');
@ -79,7 +79,7 @@ export async function installBins() {
.map((bin) => bin.trim())
.filter(Boolean);
if (CACHE_ENABLED) {
if (isCacheEnabled()) {
bins.push('cargo-cache');
}
@ -98,61 +98,6 @@ export async function installBins() {
await exec.exec('cargo', ['binstall', '--no-confirm', '--log-level', 'info', ...bins]);
}
export function getCacheTarget(): string {
return core.getInput('cache-target') || 'debug';
}
export function getCachePaths(): string[] {
return [
// ~/.cargo/registry
path.join(CARGO_HOME, 'registry'),
// /workspace/target/debug
path.join(WORKSPACE_ROOT, 'target', getCacheTarget()),
];
}
export function getCachePrefixes(): string[] {
return [`setup-rustcargo-v1-${process.platform}`, 'setup-rustcargo-v1'];
}
export async function getPrimaryCacheKey() {
const hasher = crypto.createHash('sha1');
core.info('Generating cache key');
core.debug(`Hashing Rust version = ${RUST_VERSION}`);
hasher.update(RUST_VERSION);
core.debug(`Hashing Rust commit hash = ${RUST_HASH}`);
hasher.update(RUST_HASH);
const lockHash = await glob.hashFiles('Cargo.lock');
core.debug(`Hashing Cargo.lock = ${lockHash}`);
hasher.update(lockHash);
const cacheTarget = getCacheTarget();
core.debug(`Hashing target profile = ${cacheTarget}`);
hasher.update(cacheTarget);
const workflow = process.env.GITHUB_WORKFLOW;
if (workflow) {
core.debug(`Hashing GITHUB_WORKFLOW = ${workflow}`);
hasher.update(workflow);
}
const job = process.env.GITHUB_JOB;
if (job) {
core.debug(`Hashing GITHUB_JOB = ${job}`);
hasher.update(job);
}
return `${getCachePrefixes()[0]}-${hasher.digest('hex')}`;
}
export async function cleanCargoRegistry() {
core.info('Cleaning ~/.cargo before saving');
@ -208,7 +153,7 @@ export async function cleanTargetProfile() {
}
export async function saveCache() {
if (!CACHE_ENABLED) {
if (!isCacheEnabled()) {
return;
}
@ -229,7 +174,7 @@ export async function saveCache() {
}
export async function restoreCache() {
if (!CACHE_ENABLED) {
if (!isCacheEnabled()) {
return;
}