From b3ad80abe3e79976564272ca5534ddfd0e0b7536 Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 15:29:01 +0530 Subject: [PATCH] add lock awareness --- README.md | 12 ++--- __test__/warpbuild-mirror.test.ts | 73 +++++++++++++++++++++++++++++++ dist/index.js | 38 ++++++++++++++-- src/warpbuild/backend-api.ts | 8 +++- src/warpbuild/mirror-cache.ts | 34 ++++++++++++-- 5 files changed, 152 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e2e3c27..89eea5d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # WarpBuild Checkout This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in -replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's -bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git -alternates, so the fetch from GitHub downloads only the delta instead of the whole -repository. On a cache miss, the run hydrates the mirror once (full clone + upload); -mirrors expire on a server-configured TTL and re-hydrate. +replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` +objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA. +A later job checking out the same commit restores that snapshot and skips the fetch from +GitHub entirely — cutting request load (the main source of GitHub rate-limiting on +matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. +- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS); + everything else runs exactly like upstream. - Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. - Fail-open — any cache error degrades to stock `actions/checkout` behavior. - All fork code lives in `src/warpbuild/` diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 1f75b01..cbaa272 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,8 +6,13 @@ import { afterEach, afterAll } from '@jest/globals' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' import { SKIP_NOT_WARPBUILD, + assertSafeTarMembers, computeDestinationRef, getMirrorCacheSkipReason } from '../src/warpbuild/mirror-cache.js' @@ -131,6 +136,69 @@ describe('warpbuild snapshot cache', () => { }) }) + describe('assertSafeTarMembers', () => { + async function makeTar( + dir: string, + members: string[], + writeFiles = true + ): Promise { + if (writeFiles) { + for (const m of members) { + const abs = path.join(dir, m) + await fs.promises.mkdir(path.dirname(abs), {recursive: true}) + await fs.promises.writeFile(abs, 'x') + } + } + const tar = path.join(dir, 'out.tar') + await exec.exec('tar', ['-cf', tar, '-C', dir, ...members]) + return tar + } + + it('accepts objects/ and shallow members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow']) + await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined() + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects members outside objects/ and shallow (e.g. hooks)', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit']) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects path traversal members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + await fs.promises.mkdir(path.join(dir, 'sub')) + const tar = path.join(dir, 'evil.tar') + // -P preserves the ../ member instead of stripping it. + await fs.promises.writeFile(path.join(dir, 'evil'), 'x') + await exec.exec('tar', [ + '-cPf', + tar, + '-C', + path.join(dir, 'sub'), + '../evil' + ]) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + }) + describe('computeDestinationRef', () => { it('maps the refs the fetch would have created', () => { expect(computeDestinationRef('refs/heads/main')).toBe( @@ -172,6 +240,11 @@ describe('warpbuild snapshot cache', () => { expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') }) + it('maps 409 upload responses to locked (another job is uploading)', async () => { + stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) + expect((await requestUploadURL('123', SHA)).kind).toBe('locked') + }) + it('maps 403 to disabled (backend kill switch)', async () => { stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') diff --git a/dist/index.js b/dist/index.js index 1cfe740..c91eee1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41699,7 +41699,8 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. +// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is +// uploading this snapshot (skip upload); else = fall back. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41753,6 +41754,10 @@ async function requestUploadURL(repoKey, sha) { core_debug(`[wb-cache] upload-url answered 403 (disabled)`); return { kind: 'disabled' }; } + if (res.status === 409) { + core_debug(`[wb-cache] upload-url answered 409 (locked)`); + return { kind: 'locked' }; + } core_debug(`[wb-cache] upload-url answered ${res.status}`); return { kind: 'error' }; } @@ -41910,6 +41915,28 @@ async function contribute(settings) { endGroup(); } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +async function assertSafeTarMembers(tar) { + let listing = ''; + await exec_exec('tar', ['-tf', tar], { + silent: true, + listeners: { stdout: (d) => (listing += d.toString()) } + }); + for (const raw of listing.split('\n')) { + const member = raw.trim(); + if (!member) { + continue; + } + const top = member.replace(/^\.\//, '').split('/')[0]; + if (member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow')) { + throw new Error(`unexpected snapshot tar member: ${member}`); + } + } +} async function restoreSnapshot(settings, url, sha) { const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); @@ -41921,6 +41948,7 @@ async function restoreSnapshot(settings, url, sha) { throw new Error(`snapshot download answered ${res.status}`); } await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await assertSafeTarMembers(tmpTar); await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); if (check !== 0) { @@ -41974,9 +42002,11 @@ async function uploadSnapshot(settings) { const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; const upload = await requestUploadURL(repoKey, sha); if (upload.kind !== 'ok') { - info(upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading'); + info(upload.kind === 'locked' + ? 'Another job is already uploading this snapshot; skipping' + : upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading'); return; } const init = { diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index d51cd06..5f6ca85 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -3,7 +3,8 @@ import * as core from '@actions/core' // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. +// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is +// uploading this snapshot (skip upload); else = fall back. const API_TIMEOUT_MS = 10_000 @@ -22,6 +23,7 @@ export type SnapshotLookup = export type UploadURLResult = | {kind: 'ok'; url: string} | {kind: 'disabled'} + | {kind: 'locked'} | {kind: 'error'} function baseUrl(): string { @@ -89,6 +91,10 @@ export async function requestUploadURL( core.debug(`[wb-cache] upload-url answered 403 (disabled)`) return {kind: 'disabled'} } + if (res.status === 409) { + core.debug(`[wb-cache] upload-url answered 409 (locked)`) + return {kind: 'locked'} + } core.debug(`[wb-cache] upload-url answered ${res.status}`) return {kind: 'error'} } catch (error) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 9e604e1..dbeb476 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -171,6 +171,31 @@ export async function contribute(settings: IGitSourceSettings): Promise { } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +export async function assertSafeTarMembers(tar: string): Promise { + let listing = '' + await exec.exec('tar', ['-tf', tar], { + silent: true, + listeners: {stdout: (d: Buffer) => (listing += d.toString())} + }) + for (const raw of listing.split('\n')) { + const member = raw.trim() + if (!member) { + continue + } + const top = member.replace(/^\.\//, '').split('/')[0] + if ( + member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow') + ) { + throw new Error(`unexpected snapshot tar member: ${member}`) + } + } +} + async function restoreSnapshot( settings: IGitSourceSettings, url: string, @@ -189,6 +214,7 @@ async function restoreSnapshot( Readable.fromWeb(res.body as import('stream/web').ReadableStream), fs.createWriteStream(tmpTar) ) + await assertSafeTarMembers(tmpTar) await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) const check = await exec.exec( @@ -253,9 +279,11 @@ async function uploadSnapshot(settings: IGitSourceSettings): Promise { const upload = await api.requestUploadURL(repoKey, sha) if (upload.kind !== 'ok') { core.info( - upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading' + upload.kind === 'locked' + ? 'Another job is already uploading this snapshot; skipping' + : upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading' ) return }