migrate to post job uploads

This commit is contained in:
darshanime 2026-07-09 14:07:20 +05:30
parent 8b5fffd5f3
commit 41e5319596
6 changed files with 183 additions and 253 deletions

View file

@ -2,7 +2,7 @@
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA.
A later job checking out the same commit restores that snapshot and skips the fetch from
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.

View file

@ -14,9 +14,11 @@ import {
SKIP_NOT_WARPBUILD,
assertSafeTarMembers,
computeDestinationRef,
getMirrorCacheSkipReason
getMirrorCacheSkipReason,
manifestDir,
resetGitObjects
} from '../src/warpbuild/mirror-cache.js'
import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js'
import {lookupSnapshot} from '../src/warpbuild/backend-api.js'
import {IGitSourceSettings} from '../src/git-source-settings.js'
const WB_ENV = [
@ -136,6 +138,60 @@ describe('warpbuild snapshot cache', () => {
})
})
describe('resetGitObjects', () => {
it('restores the empty git-init object layout, dropping restored junk', async () => {
const gitDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-reset-')
)
try {
// Simulate a partially-restored .git: stray objects, a pack, a shallow file.
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
await fs.promises.writeFile(
path.join(gitDir, 'objects', 'ab'),
'junk loose object'
)
await fs.promises.writeFile(
path.join(gitDir, 'objects', 'pack', 'pack-x.pack'),
'junk pack'
)
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
await resetGitObjects(gitDir)
// shallow gone; objects/ holds only the two empty init dirs.
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack'])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'pack'))
).toEqual([])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'info'))
).toEqual([])
} finally {
await fs.promises.rm(gitDir, {recursive: true, force: true})
}
})
it('is a no-op-safe on an already-clean .git', async () => {
const gitDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-reset-')
)
try {
await expect(resetGitObjects(gitDir)).resolves.toBeUndefined()
expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack'])
} finally {
await fs.promises.rm(gitDir, {recursive: true, force: true})
}
})
})
describe('assertSafeTarMembers', () => {
async function makeTar(
dir: string,
@ -235,20 +291,14 @@ describe('warpbuild snapshot cache', () => {
expect(lastUrl).toContain(`sha=${SHA}`)
})
it('maps 404 to miss (upload after the stock fetch)', async () => {
it('maps 404 to miss (agent uploads after the job)', async () => {
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
})
it('maps 409 upload responses to locked (another job is uploading)', async () => {
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
})
it('maps 403 to disabled (backend kill switch)', async () => {
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
})
it('maps other statuses and network failures to error', async () => {
@ -258,15 +308,24 @@ describe('warpbuild snapshot cache', () => {
throw new Error('boom')
}) as typeof fetch
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
expect((await requestUploadURL('123', SHA)).kind).toBe('error')
})
})
describe('manifestDir', () => {
const saved = process.env['RUNNER_TEMP']
afterEach(() => {
if (saved === undefined) {
delete process.env['RUNNER_TEMP']
} else {
process.env['RUNNER_TEMP'] = saved
}
})
it('maps 200 upload responses to ok', async () => {
stubFetch(200, {url: 'https://s3/put'})
expect(await requestUploadURL('123', SHA)).toEqual({
kind: 'ok',
url: 'https://s3/put'
})
it('places manifests under RUNNER_TEMP/wb-snapshots', () => {
process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x')
expect(manifestDir()).toBe(
path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots')
)
})
})
})

View file

@ -112,6 +112,8 @@ outputs:
description: 'The branch, tag or SHA that was checked out'
commit:
description: 'The commit SHA that was checked out'
cache-hit:
description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)'
runs:
using: node24
main: dist/index.js

154
dist/index.js vendored
View file

@ -41697,10 +41697,9 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
/* eslint-disable i18n-text/no-en -- upstream convention */
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
// uploading this snapshot (skip upload); else = fall back.
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
const API_TIMEOUT_MS = 10_000;
function backend_api_baseUrl() {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@ -41732,40 +41731,6 @@ async function lookupSnapshot(repoKey, sha) {
return { kind: 'error' };
}
}
async function requestUploadURL(repoKey, sha) {
try {
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify({ repo_key: repoKey, sha }),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
});
if (res.status === 200) {
const body = (await res.json());
if (body.url) {
return { kind: 'ok', url: body.url };
}
return { kind: 'error' };
}
if (res.status === 403) {
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
return { kind: 'disabled' };
}
if (res.status === 409) {
core_debug(`[wb-cache] upload-url answered 409 (locked)`);
return { kind: 'locked' };
}
core_debug(`[wb-cache] upload-url answered ${res.status}`);
return { kind: 'error' };
}
catch (error) {
core_debug(`[wb-cache] upload-url failed: ${error}`);
return { kind: 'error' };
}
}
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
@ -41782,9 +41747,6 @@ async function requestUploadURL(repoKey, sha) {
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
const CACHE_STATE_FILE = 'wb-cache-state';
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
@ -41849,16 +41811,17 @@ function computeDestinationRef(ref) {
return null;
}
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
// Sets the `cache-hit` action output (true only on a restore).
async function setup(settings) {
const restored = await setupImpl(settings);
setOutput('cache-hit', restored ? 'true' : 'false');
return restored;
}
async function setupImpl(settings) {
decision = 'off';
const skipReason = getMirrorCacheSkipReason(settings);
if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) {
core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
}
else {
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
}
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
return false;
}
startGroup('WarpBuild: checkout snapshot cache');
@ -41902,20 +41865,17 @@ async function setupInner(settings) {
info('Snapshot restored; skipping the GitHub fetch entirely');
return true;
}
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
async function contribute(settings) {
if (decision !== 'miss') {
return;
}
startGroup('WarpBuild: uploading checkout snapshot');
try {
await uploadSnapshot(settings);
await writeSnapshotManifest(settings);
}
catch (error) {
warning(`Snapshot upload skipped: ${error}`);
}
finally {
endGroup();
warning(`Snapshot manifest not written: ${error}`);
}
}
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
@ -41968,70 +41928,50 @@ async function restoreSnapshot(settings, url, sha) {
sha
]);
}
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
return true;
}
catch (error) {
warning(`Snapshot restore failed: ${error}`);
// Reset .git to its freshly-init state.
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
recursive: true,
force: true
});
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
recursive: true
});
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
recursive: true
});
await resetGitObjects(gitDir);
return false;
}
finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
}
}
async function uploadSnapshot(settings) {
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
// creates, so a failed restore leaves no trace for the fallback fetch.
async function resetGitObjects(gitDir) {
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
recursive: true,
force: true
});
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
recursive: true
});
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
recursive: true
});
}
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
function manifestDir() {
const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir();
return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots');
}
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(settings) {
const sha = settings.commit;
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`);
try {
const members = ['objects'];
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) {
members.push('shallow');
}
await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]);
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
const upload = await requestUploadURL(repoKey, sha);
if (upload.kind !== 'ok') {
info(upload.kind === 'locked'
? 'Another job is already uploading this snapshot; skipping'
: upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading');
return;
}
const init = {
method: 'PUT',
headers: {
'content-length': String(size),
'content-type': 'application/x-tar'
},
body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)),
duplex: 'half',
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
};
const res = await fetch(upload.url, init);
if (!res.ok) {
throw new Error(`snapshot upload answered ${res.status}`);
}
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
}
finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
}
const manifest = {
repo_key: process.env['GITHUB_REPOSITORY_ID'],
sha,
git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git')
};
const dir = manifestDir();
await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true });
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest));
info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`);
}
;// CONCATENATED MODULE: ./src/git-source-provider.ts

View file

@ -1,10 +1,9 @@
/* eslint-disable i18n-text/no-en -- upstream convention */
import * as core from '@actions/core'
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
// uploading this snapshot (skip upload); else = fall back.
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
const API_TIMEOUT_MS = 10_000
@ -20,12 +19,6 @@ export type SnapshotLookup =
| {kind: 'disabled'}
| {kind: 'error'}
export type UploadURLResult =
| {kind: 'ok'; url: string}
| {kind: 'disabled'}
| {kind: 'locked'}
| {kind: 'error'}
function baseUrl(): string {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
}
@ -65,40 +58,3 @@ export async function lookupSnapshot(
return {kind: 'error'}
}
}
export async function requestUploadURL(
repoKey: string,
sha: string
): Promise<UploadURLResult> {
try {
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify({repo_key: repoKey, sha}),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
})
if (res.status === 200) {
const body = (await res.json()) as {url?: string}
if (body.url) {
return {kind: 'ok', url: body.url}
}
return {kind: 'error'}
}
if (res.status === 403) {
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
return {kind: 'disabled'}
}
if (res.status === 409) {
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
return {kind: 'locked'}
}
core.debug(`[wb-cache] upload-url answered ${res.status}`)
return {kind: 'error'}
} catch (error) {
core.debug(`[wb-cache] upload-url failed: ${error}`)
return {kind: 'error'}
}
}

View file

@ -15,10 +15,6 @@ import * as api from './backend-api.js'
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
const UPLOAD_TIMEOUT_MS = 15 * 60_000
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
export const CACHE_STATE_FILE = 'wb-cache-state'
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
export const SKIP_NOT_WARPBUILD =
@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null {
}
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
// Sets the `cache-hit` action output (true only on a restore).
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
const restored = await setupImpl(settings)
core.setOutput('cache-hit', restored ? 'true' : 'false')
return restored
}
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> {
decision = 'off'
const skipReason = getMirrorCacheSkipReason(settings)
if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) {
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
} else {
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
}
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
return false
}
core.startGroup('WarpBuild: checkout snapshot cache')
@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
return true
}
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
export async function contribute(settings: IGitSourceSettings): Promise<void> {
if (decision !== 'miss') {
return
}
core.startGroup('WarpBuild: uploading checkout snapshot')
try {
await uploadSnapshot(settings)
await writeSnapshotManifest(settings)
} catch (error) {
core.warning(`Snapshot upload skipped: ${error}`)
} finally {
core.endGroup()
core.warning(`Snapshot manifest not written: ${error}`)
}
}
@ -242,80 +239,56 @@ async function restoreSnapshot(
])
}
await fs.promises.writeFile(
path.join(gitDir, CACHE_STATE_FILE),
`hit ${sha}\n`
)
return true
} catch (error) {
core.warning(`Snapshot restore failed: ${error}`)
// Reset .git to its freshly-init state.
await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true,
force: true
})
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
recursive: true
})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
await resetGitObjects(gitDir)
return false
} finally {
await fs.promises.rm(tmpTar, {force: true})
}
}
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
const sha = settings.commit
const gitDir = path.join(settings.repositoryPath, '.git')
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
try {
const members = ['objects']
if (fs.existsSync(path.join(gitDir, 'shallow'))) {
members.push('shallow')
}
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
const size = (await fs.promises.stat(tmpTar)).size
const upload = await api.requestUploadURL(repoKey, sha)
if (upload.kind !== 'ok') {
core.info(
upload.kind === 'locked'
? 'Another job is already uploading this snapshot; skipping'
: upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading'
)
return
}
const init: RequestInit & {duplex: 'half'} = {
method: 'PUT',
headers: {
'content-length': String(size),
'content-type': 'application/x-tar'
},
body: Readable.toWeb(
fs.createReadStream(tmpTar)
) as unknown as ReadableStream,
duplex: 'half',
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
}
const res = await fetch(upload.url, init)
if (!res.ok) {
throw new Error(`snapshot upload answered ${res.status}`)
}
await fs.promises.writeFile(
path.join(gitDir, CACHE_STATE_FILE),
`uploaded ${sha}\n`
)
core.info(
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
)
} finally {
await fs.promises.rm(tmpTar, {force: true})
}
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
// creates, so a failed restore leaves no trace for the fallback fetch.
export async function resetGitObjects(gitDir: string): Promise<void> {
await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true,
force: true
})
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
recursive: true
})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
}
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
export function manifestDir(): string {
const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
return path.join(runnerTemp, 'wb-snapshots')
}
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(
settings: IGitSourceSettings
): Promise<void> {
const sha = settings.commit
const manifest = {
repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
sha,
git_dir: path.join(settings.repositoryPath, '.git')
}
const dir = manifestDir()
await fs.promises.mkdir(dir, {recursive: true})
await fs.promises.writeFile(
path.join(dir, `${sha}.json`),
JSON.stringify(manifest)
)
core.info(
`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
)
}