From 8879f313691256e4a8289ab3154e08e51c12ebc3 Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 09:40:20 +0530 Subject: [PATCH 01/13] init warpbuild caching --- README.md | 15 ++ __test__/warpbuild-mirror.test.ts | 197 +++++++++++++++++ dist/index.js | 345 ++++++++++++++++++++++++++++++ src/git-source-provider.ts | 5 + src/warpbuild/backend-api.ts | 101 +++++++++ src/warpbuild/mirror-cache.ts | 326 ++++++++++++++++++++++++++++ 6 files changed, 989 insertions(+) create mode 100644 __test__/warpbuild-mirror.test.ts create mode 100644 src/warpbuild/backend-api.ts create mode 100644 src/warpbuild/mirror-cache.ts diff --git a/README.md b/README.md index 5509e7d..e2e3c27 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ +# WarpBuild Checkout + +This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in +replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's +bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git +alternates, so the fetch from GitHub downloads only the delta instead of the whole +repository. On a cache miss, the run hydrates the mirror once (full clone + upload); +mirrors expire on a server-configured TTL and re-hydrate. + +- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. +- Fail-open — any cache error degrades to stock `actions/checkout` behavior. +- All fork code lives in `src/warpbuild/` + +--- + [![Build and Test](https://github.com/actions/checkout/actions/workflows/test.yml/badge.svg)](https://github.com/actions/checkout/actions/workflows/test.yml) # Checkout v7 diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts new file mode 100644 index 0000000..023b257 --- /dev/null +++ b/__test__/warpbuild-mirror.test.ts @@ -0,0 +1,197 @@ +import { + describe, + it, + expect, + beforeEach, + afterEach, + afterAll +} from '@jest/globals' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { + ALTERNATES_CONTENT, + SKIP_NOT_WARPBUILD, + mirrorPath, + getMirrorCacheSkipReason, + writeAlternates +} from '../src/warpbuild/mirror-cache.js' +import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {IGitSourceSettings} from '../src/git-source-settings.js' + +const WB_ENV = [ + 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', + 'WARPBUILD_HOST_URL', + 'GITHUB_REPOSITORY_ID', + 'GITHUB_REPOSITORY' +] +const savedEnv: {[key: string]: string | undefined} = {} +for (const key of WB_ENV) { + savedEnv[key] = process.env[key] +} + +function settingsFor( + owner: string, + repo: string, + serverUrl?: string +): IGitSourceSettings { + return { + repositoryOwner: owner, + repositoryName: repo, + repositoryPath: '/tmp/does-not-matter', + githubServerUrl: serverUrl + } as unknown as IGitSourceSettings +} + +function setWarpBuildEnv(): void { + process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token' + process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' + process.env['GITHUB_REPOSITORY_ID'] = '123456789' + process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' +} + +describe('warpbuild mirror cache', () => { + beforeEach(() => { + setWarpBuildEnv() + }) + + afterAll(() => { + for (const key of WB_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = savedEnv[key] + } + } + }) + + describe('getMirrorCacheSkipReason', () => { + const winOnly = process.platform === 'win32' ? it.skip : it + + winOnly('returns null for the workflow repo on a WarpBuild runner', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBeNull() + }) + + it('reports a non-WarpBuild runner', () => { + delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe(SKIP_NOT_WARPBUILD) + }) + + it('reports repository: inputs that are not the workflow repo', () => { + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) + ).toBe( + "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" + ) + }) + + it('reports a missing GITHUB_REPOSITORY_ID', () => { + delete process.env['GITHUB_REPOSITORY_ID'] + expect( + getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) + ).toBe('GITHUB_REPOSITORY_ID is not set') + }) + + winOnly('accepts explicit github.com server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://github.com') + ) + ).toBeNull() + }) + + it('reports GHES server urls', () => { + expect( + getMirrorCacheSkipReason( + settingsFor('octocat', 'hello-world', 'https://ghes.example.com') + ) + ).toBe("server 'https://ghes.example.com' is not github.com") + }) + }) + + describe('backend api http contract', () => { + const realFetch = globalThis.fetch + + afterEach(() => { + globalThis.fetch = realFetch + }) + + function stubFetch(status: number, body: unknown): void { + globalThis.fetch = (async () => + new Response(JSON.stringify(body), {status})) as typeof fetch + } + + it('maps 200 to hit', async () => { + stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) + const result = await lookupMirror('123') + expect(result.kind).toBe('hit') + if (result.kind === 'hit') { + expect(result.info.url).toBe('https://s3/x') + } + }) + + it('maps 404 to miss (hydrate)', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupMirror('123')).kind).toBe('miss') + }) + + it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupMirror('123')).kind).toBe('disabled') + expect((await requestUploadURL('123')).kind).toBe('disabled') + }) + + it('maps other statuses to error (fall back without hydrating)', async () => { + stubFetch(500, {}) + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps network failures to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupMirror('123')).kind).toBe('error') + expect((await requestUploadURL('123')).kind).toBe('error') + }) + + it('maps 200 upload responses to ok', async () => { + stubFetch(200, {url: 'https://s3/put'}) + const result = await requestUploadURL('123') + expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) + }) + }) + + describe('mirror layout', () => { + it('keeps the mirror inside .git', () => { + expect(mirrorPath('/work/repo')).toBe( + path.join('/work/repo', '.git', 'wb-mirror.git') + ) + }) + + it('uses a relative alternates path that never leaves .git', () => { + expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n') + expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false) + }) + + it('writeAlternates writes the relative path into objects/info', async () => { + const workspace = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-mirror-test-') + ) + try { + await writeAlternates(workspace) + const content = await fs.promises.readFile( + path.join(workspace, '.git', 'objects', 'info', 'alternates'), + 'utf8' + ) + expect(content).toBe(ALTERNATES_CONTENT) + } finally { + await fs.promises.rm(workspace, {recursive: true, force: true}) + } + }) + }) +}) diff --git a/dist/index.js b/dist/index.js index b381bd2..0882dfb 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41692,6 +41692,347 @@ function ref_helper_select(obj, path) { return ref_helper_select(obj[key], path.substr(i + 1)); } +;// CONCATENATED MODULE: external "stream/promises" +const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); +;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download +const API_TIMEOUT_MS = 10_000; +function backend_api_baseUrl() { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); +} +function authHeader() { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; +} +async function lookupMirror(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { + headers: { authorization: authHeader() }, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + return { kind: 'hit', info: (await res.json()) }; + } + if (res.status === 404) { + return { kind: 'miss' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] download-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] download-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] download-url failed: ${error}`); + return { kind: 'error' }; + } +} +async function requestUploadURL(repoKey) { + try { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({ repo_key: repoKey }), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const body = (await res.json()); + if (body.url) { + return { kind: 'ok', url: body.url }; + } + return { kind: 'error' }; + } + if (res.status === 403) { + core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); + return { kind: 'disabled' }; + } + core_debug(`[wb-mirror] upload-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-mirror] upload-url failed: ${error}`); + return { kind: 'error' }; + } +} + +;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ + + + + + + + + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. +const MIRROR_DIR = 'wb-mirror.git'; +const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`; +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; +function mirrorPath(repositoryPath) { + return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); +} +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +function getMirrorCacheSkipReason(settings) { + // Only on WarpBuild runners (these are injected into every job's env there). + if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL']) { + return SKIP_NOT_WARPBUILD; + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet'; + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set'; + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); + if (server !== 'https://github.com') { + return `server '${server}' is not github.com`; + } + return null; +} +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +async function setup(settings, repositoryUrl) { + const skipReason = getMirrorCacheSkipReason(settings); + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); + } + else { + info(`WarpBuild mirror cache skipped: ${skipReason}`); + } + return; + } + startGroup('WarpBuild: setting up git mirror cache'); + try { + await setupInner(settings, repositoryUrl); + } + catch (error) { + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + } + finally { + endGroup(); + } +} +async function setupInner(settings, repositoryUrl) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const mirror = mirrorPath(settings.repositoryPath); + // A second checkout of the same repo in one job finds the mirror already in place. + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) { + info('Mirror already present, reusing it'); + await writeAlternates(settings.repositoryPath); + return; + } + const lookup = await lookupMirror(repoKey); + if (lookup.kind === 'disabled') { + info('Mirror cache is disabled by the backend for this organization; using standard checkout'); + return; + } + if (lookup.kind === 'error') { + info('Mirror cache backend unavailable; using standard checkout'); + return; + } + if (lookup.kind === 'hit') { + info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath); + info('Mirror restored; the fetch below downloads only the delta'); + } + // Restore failure: fall through to plain checkout. The object exists, so the + // failure was transfer-shaped — re-downloading would just repeat the pain. + return; + } + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await requestUploadURL(repoKey); + if (probe.kind !== 'ok') { + info(probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation'); + return; + } + info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); + await createMirrorFromGitHub(settings, repositoryUrl, mirror); + await writeAlternates(settings.repositoryPath); + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror); +} +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +async function writeAlternates(repositoryPath) { + const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); + await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); + info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); +} +async function restoreMirror(url, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); + await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); + return true; + } + catch (error) { + warning(`Mirror restore failed: ${error}`); + // Never leave a partial mirror behind an alternates file — that is corruption. + await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); + return false; + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub(settings, repositoryUrl, mirror) { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64'); + core_setSecret(basicCredential); + const env = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value; + } + } + env['GIT_CONFIG_COUNT'] = '1'; + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'; + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`; + await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env }); + await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], { + env + }); + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec_exec('git', [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], { env }); +} +async function uploadMirror(repoKey, mirror) { + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`); + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec_exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]); + const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await requestUploadURL(repoKey); + if (fresh.kind !== 'ok') { + throw new Error(fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable'); + } + const init = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)), + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + }; + const res = await fetch(fresh.url, init); + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`); + } + info(`Mirror uploaded (${size} bytes); future runs will restore it`); + } + catch (error) { + warning(`Mirror upload skipped: ${error}`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + } +} + ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -41705,6 +42046,7 @@ function ref_helper_select(obj, path) { + async function getSource(settings) { // Repository URL info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`); @@ -41774,6 +42116,9 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await setup(settings, repositoryUrl); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index b9c1d35..eedeb7f 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -9,6 +9,7 @@ import * as path from 'path' import * as refHelper from './ref-helper.js' import * as stateHelper from './state-helper.js' import * as urlHelper from './url-helper.js' +import * as warpbuildMirror from './warpbuild/mirror-cache.js' import { MinimumGitSparseCheckoutVersion, IGitCommandManager @@ -130,6 +131,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.init(objectFormat) await git.remoteAdd('origin', repositoryUrl) core.endGroup() + + // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and + // point alternates at it so the fetch below downloads only the delta from GitHub. + await warpbuildMirror.setup(settings, repositoryUrl) } // Disable automatic garbage collection diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts new file mode 100644 index 0000000..23eb8a1 --- /dev/null +++ b/src/warpbuild/backend-api.ts @@ -0,0 +1,101 @@ +/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +import * as core from '@actions/core' + +// Thin client for backend-core's /api/v1/git-mirrors endpoints. +// +// Auth is the runner verification token every WarpBuild job carries in its env +// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the +// token alone. +// +// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): +// 200 -> use the presigned URL +// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) +// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror +// creation and upload entirely, behave exactly like stock actions/checkout +// else -> transient trouble: fall back WITHOUT the mirror download + +const API_TIMEOUT_MS = 10_000 + +export interface MirrorDownloadInfo { + url: string + size_bytes: number + created_at: string +} + +export type MirrorLookup = + | {kind: 'hit'; info: MirrorDownloadInfo} + | {kind: 'miss'} + | {kind: 'disabled'} + | {kind: 'error'} + +export type UploadURLResult = + | {kind: 'ok'; url: string} + | {kind: 'disabled'} + | {kind: 'error'} + +function baseUrl(): string { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') +} + +function authHeader(): string { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` +} + +export async function lookupMirror(repoKey: string): Promise { + try { + const res = await fetch( + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, + { + headers: {authorization: authHeader()}, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + } + ) + if (res.status === 200) { + return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} + } + if (res.status === 404) { + return {kind: 'miss'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] download-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] download-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] download-url failed: ${error}`) + return {kind: 'error'} + } +} + +export async function requestUploadURL( + repoKey: string +): Promise { + try { + const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify({repo_key: repoKey}), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const body = (await res.json()) as {url?: string} + if (body.url) { + return {kind: 'ok', url: body.url} + } + return {kind: 'error'} + } + if (res.status === 403) { + core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) + return {kind: 'disabled'} + } + core.debug(`[wb-mirror] upload-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-mirror] upload-url failed: ${error}`) + return {kind: 'error'} + } +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts new file mode 100644 index 0000000..03f5f91 --- /dev/null +++ b/src/warpbuild/mirror-cache.ts @@ -0,0 +1,326 @@ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and + .js-suffixed ESM imports both follow upstream's own conventions; the import plugin + has no TS resolver configured in this repo. */ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {Readable} from 'stream' +import {pipeline} from 'stream/promises' +import {IGitSourceSettings} from '../git-source-settings.js' +import * as api from './backend-api.js' + +// WarpBuild git-mirror cache. +// +// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we +// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at +// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and +// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers +// 404), THIS run creates the mirror: one full download of all branches + tags from +// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is +// disabled for this org (backend-driven kill switch) and we skip everything — no +// mirror creation, no upload. +// +// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path +// is relative, so it survives every container mount scheme: `container:` jobs (/__w), +// Docker container actions (/github/workspace), and `docker build COPY .`. +// +// Everything here is fail-open: any error or timeout degrades to stock actions/checkout +// behavior with a warning, never a failed checkout. + +const MIRROR_DIR = 'wb-mirror.git' +export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n` + +const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 + +// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state +// everywhere outside WarpBuild); every other reason is logged at info. +export const SKIP_NOT_WARPBUILD = + 'not running on a WarpBuild runner (WARPBUILD_* env not present)' + +export function mirrorPath(repositoryPath: string): string { + return path.join(repositoryPath, '.git', MIRROR_DIR) +} + +// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache +// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and +// deliberately strict: anything unexpected means "behave exactly like upstream". +export function getMirrorCacheSkipReason( + settings: IGitSourceSettings +): string | null { + // Only on WarpBuild runners (these are injected into every job's env there). + if ( + !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL'] + ) { + return SKIP_NOT_WARPBUILD + } + // Linux + macOS in v1. + if (process.platform === 'win32') { + return 'Windows is not supported by the mirror cache yet' + } + // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only + // cache when that is what is being checked out (`repository:` inputs fall back). + const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' + if (!repoKey) { + return 'GITHUB_REPOSITORY_ID is not set' + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` + } + // github.com only (repo ids and mirror keys assume it). + const server = (settings.githubServerUrl || 'https://github.com').replace( + /\/+$/, + '' + ) + if (server !== 'https://github.com') { + return `server '${server}' is not github.com` + } + return null +} + +// setup is the single upstream splice point, called right after `git init` + +// `git remote add` for a fresh repository. It never throws. +export async function setup( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const skipReason = getMirrorCacheSkipReason(settings) + if (skipReason) { + if (skipReason === SKIP_NOT_WARPBUILD) { + core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) + } else { + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + } + return + } + core.startGroup('WarpBuild: setting up git mirror cache') + try { + await setupInner(settings, repositoryUrl) + } catch (error) { + core.warning( + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + ) + } finally { + core.endGroup() + } +} + +async function setupInner( + settings: IGitSourceSettings, + repositoryUrl: string +): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const mirror = mirrorPath(settings.repositoryPath) + + // A second checkout of the same repo in one job finds the mirror already in place. + if (fs.existsSync(path.join(mirror, 'objects'))) { + core.info('Mirror already present, reusing it') + await writeAlternates(settings.repositoryPath) + return + } + + const lookup = await api.lookupMirror(repoKey) + + if (lookup.kind === 'disabled') { + core.info( + 'Mirror cache is disabled by the backend for this organization; using standard checkout' + ) + return + } + + if (lookup.kind === 'error') { + core.info('Mirror cache backend unavailable; using standard checkout') + return + } + + if (lookup.kind === 'hit') { + core.info( + `Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` + ) + if (await restoreMirror(lookup.info.url, mirror)) { + await writeAlternates(settings.repositoryPath) + core.info('Mirror restored; the fetch below downloads only the delta') + } + return + } + + // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or + // unreachable backend never costs a wasted full mirror clone. + const probe = await api.requestUploadURL(repoKey) + if (probe.kind !== 'ok') { + core.info( + probe.kind === 'disabled' + ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' + : 'Mirror cache backend unavailable; skipping mirror creation' + ) + return + } + + core.info( + 'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' + ) + await createMirrorFromGitHub(settings, repositoryUrl, mirror) + await writeAlternates(settings.repositoryPath) + // Upload failures only warn: the local mirror still accelerates THIS run. + await uploadMirror(repoKey, mirror) +} + +// writeAlternates points the workspace repo's object lookups at the mirror. The path is +// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. +export async function writeAlternates(repositoryPath: string): Promise { + const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') + await fs.promises.mkdir(infoDir, {recursive: true}) + await fs.promises.writeFile( + path.join(infoDir, 'alternates'), + ALTERNATES_CONTENT + ) + core.info( + `Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` + ) +} + +async function restoreMirror(url: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`mirror download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(tmpTar) + ) + await fs.promises.mkdir(mirror, {recursive: true}) + await exec.exec('tar', ['-xf', tmpTar, '-C', mirror]) + return true + } catch (error) { + core.warning(`Mirror restore failed: ${error}`) + // Never leave a partial mirror behind an alternates file — that is corruption. + await fs.promises.rm(mirror, {recursive: true, force: true}) + return false + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} + +// createMirrorFromGitHub builds the bare mirror by downloading the repository from +// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. +// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also +// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large +// share of the download on PR-heavy repos). PR-triggered checkouts still work: their +// head SHA simply arrives as a small delta in the workspace fetch. +// +// The full history download itself cannot be safely avoided: a shallow or partial +// mirror behind an alternates file is an incomplete object store that git assumes is +// complete — the exact corruption Blacksmith hit and reverted. A mirror must be +// complete with respect to the refs it advertises. +async function createMirrorFromGitHub( + settings: IGitSourceSettings, + repositoryUrl: string, + mirror: string +): Promise { + // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars + // (git >= 2.31) so the credential never appears in any process's argv. + const basicCredential = Buffer.from( + `x-access-token:${settings.authToken}`, + 'utf8' + ).toString('base64') + core.setSecret(basicCredential) + + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) { + env[key] = value + } + } + env['GIT_CONFIG_COUNT'] = '1' + env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader' + env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}` + + await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env}) + await exec.exec( + 'git', + ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], + { + env + } + ) + // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. + await exec.exec( + 'git', + [ + '-c', + 'gc.auto=0', + '-C', + mirror, + 'fetch', + '--prune', + '--progress', + 'origin', + '+refs/heads/*:refs/heads/*', + '+refs/tags/*:refs/tags/*' + ], + {env} + ) +} + +async function uploadMirror(repoKey: string, mirror: string): Promise { + const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`) + try { + // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic + // trims; the tar stays a valid bare repo either way. + await exec.exec('tar', [ + '-cf', + tmpTar, + '-C', + mirror, + '--exclude', + './hooks', + '--exclude', + './description', + '--exclude', + './FETCH_HEAD', + '.' + ]) + const size = (await fs.promises.stat(tmpTar)).size + + // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the + // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. + const fresh = await api.requestUploadURL(repoKey) + if (fresh.kind !== 'ok') { + throw new Error( + fresh.kind === 'disabled' + ? 'mirror cache was disabled by the backend' + : 'upload-url unavailable' + ) + } + + const init: RequestInit & {duplex: 'half'} = { + method: 'PUT', + headers: { + 'content-length': String(size), + 'content-type': 'application/x-tar' + }, + body: Readable.toWeb( + fs.createReadStream(tmpTar) + ) as unknown as ReadableStream, + duplex: 'half', + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + } + const res = await fetch(fresh.url, init) + if (!res.ok) { + throw new Error(`mirror upload answered ${res.status}`) + } + core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) + } catch (error) { + core.warning(`Mirror upload skipped: ${error}`) + } finally { + await fs.promises.rm(tmpTar, {force: true}) + } +} From 7484b46911982521a278e7637eb2ae253367af20 Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 13:29:51 +0530 Subject: [PATCH 02/13] move to using sha uploads --- __test__/warpbuild-mirror.test.ts | 182 +++++++-------- dist/index.js | 363 ++++++++++++++--------------- src/git-source-provider.ts | 13 +- src/warpbuild/backend-api.ts | 51 ++-- src/warpbuild/mirror-cache.ts | 371 ++++++++++++++---------------- 5 files changed, 460 insertions(+), 520 deletions(-) diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 023b257..1f75b01 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,17 +6,12 @@ import { afterEach, afterAll } from '@jest/globals' -import * as fs from 'fs' -import * as os from 'os' -import * as path from 'path' import { - ALTERNATES_CONTENT, SKIP_NOT_WARPBUILD, - mirrorPath, - getMirrorCacheSkipReason, - writeAlternates + computeDestinationRef, + getMirrorCacheSkipReason } from '../src/warpbuild/mirror-cache.js' -import {lookupMirror, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js' import {IGitSourceSettings} from '../src/git-source-settings.js' const WB_ENV = [ @@ -30,16 +25,23 @@ for (const key of WB_ENV) { savedEnv[key] = process.env[key] } +const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7' + function settingsFor( - owner: string, - repo: string, - serverUrl?: string + overrides: Partial = {} ): IGitSourceSettings { return { - repositoryOwner: owner, - repositoryName: repo, + repositoryOwner: 'octocat', + repositoryName: 'hello-world', repositoryPath: '/tmp/does-not-matter', - githubServerUrl: serverUrl + ref: 'refs/heads/main', + commit: SHA, + fetchDepth: 1, + fetchTags: false, + filter: undefined, + sparseCheckout: undefined, + lfs: false, + ...overrides } as unknown as IGitSourceSettings } @@ -50,7 +52,7 @@ function setWarpBuildEnv(): void { process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' } -describe('warpbuild mirror cache', () => { +describe('warpbuild snapshot cache', () => { beforeEach(() => { setWarpBuildEnv() }) @@ -68,130 +70,130 @@ describe('warpbuild mirror cache', () => { describe('getMirrorCacheSkipReason', () => { const winOnly = process.platform === 'win32' ? it.skip : it - winOnly('returns null for the workflow repo on a WarpBuild runner', () => { - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBeNull() + winOnly('returns null for the default checkout shape', () => { + expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() }) it('reports a non-WarpBuild runner', () => { delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBe(SKIP_NOT_WARPBUILD) + expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) }) it('reports repository: inputs that are not the workflow repo', () => { expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'other-repo')) + getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) ).toBe( "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" ) }) - it('reports a missing GITHUB_REPOSITORY_ID', () => { - delete process.env['GITHUB_REPOSITORY_ID'] - expect( - getMirrorCacheSkipReason(settingsFor('octocat', 'hello-world')) - ).toBe('GITHUB_REPOSITORY_ID is not set') + it('requires an exact commit sha', () => { + expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe( + 'no exact commit sha to key on' + ) + expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe( + 'no exact commit sha to key on' + ) }) - winOnly('accepts explicit github.com server urls', () => { - expect( - getMirrorCacheSkipReason( - settingsFor('octocat', 'hello-world', 'https://github.com') - ) - ).toBeNull() + it('only serves fetch-depth 1', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( + 'fetch-depth is 0, cache only serves fetch-depth 1' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( + 'fetch-depth is 50, cache only serves fetch-depth 1' + ) }) - it('reports GHES server urls', () => { + it('skips fetch-tags, filters, sparse and lfs checkouts', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe( + 'fetch-tags is enabled' + ) + expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( + 'a fetch filter is configured' + ) expect( - getMirrorCacheSkipReason( - settingsFor('octocat', 'hello-world', 'https://ghes.example.com') - ) - ).toBe("server 'https://ghes.example.com' is not github.com") + getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) + ).toBe('sparse checkout is configured') + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( + 'lfs is enabled (lfs objects are not in the snapshot)' + ) + }) + + winOnly('accepts sha-only checkouts (empty ref)', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() + }) + + it('skips unqualified refs', () => { + expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( + "ref 'main' has no cacheable destination ref" + ) + }) + }) + + describe('computeDestinationRef', () => { + it('maps the refs the fetch would have created', () => { + expect(computeDestinationRef('refs/heads/main')).toBe( + 'refs/remotes/origin/main' + ) + expect(computeDestinationRef('refs/pull/42/merge')).toBe( + 'refs/remotes/pull/42/merge' + ) + expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3') + expect(computeDestinationRef('')).toBe('') + expect(computeDestinationRef('main')).toBeNull() }) }) describe('backend api http contract', () => { const realFetch = globalThis.fetch + let lastUrl = '' afterEach(() => { globalThis.fetch = realFetch }) function stubFetch(status: number, body: unknown): void { - globalThis.fetch = (async () => - new Response(JSON.stringify(body), {status})) as typeof fetch + globalThis.fetch = (async (input: unknown) => { + lastUrl = String(input) + return new Response(JSON.stringify(body), {status}) + }) as typeof fetch } - it('maps 200 to hit', async () => { + it('maps 200 to hit and keys by sha', async () => { stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) - const result = await lookupMirror('123') + const result = await lookupSnapshot('123', SHA) expect(result.kind).toBe('hit') - if (result.kind === 'hit') { - expect(result.info.url).toBe('https://s3/x') - } + expect(lastUrl).toContain(`sha=${SHA}`) }) - it('maps 404 to miss (hydrate)', async () => { + it('maps 404 to miss (upload after the stock fetch)', async () => { stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) - expect((await lookupMirror('123')).kind).toBe('miss') + expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') }) - it('maps 403 to disabled (backend kill switch — no hydration, no upload)', async () => { + it('maps 403 to disabled (backend kill switch)', async () => { stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) - expect((await lookupMirror('123')).kind).toBe('disabled') - expect((await requestUploadURL('123')).kind).toBe('disabled') + expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') + expect((await requestUploadURL('123', SHA)).kind).toBe('disabled') }) - it('maps other statuses to error (fall back without hydrating)', async () => { + it('maps other statuses and network failures to error', async () => { stubFetch(500, {}) - expect((await lookupMirror('123')).kind).toBe('error') - expect((await requestUploadURL('123')).kind).toBe('error') - }) - - it('maps network failures to error', async () => { + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') globalThis.fetch = (async () => { throw new Error('boom') }) as typeof fetch - expect((await lookupMirror('123')).kind).toBe('error') - expect((await requestUploadURL('123')).kind).toBe('error') + expect((await lookupSnapshot('123', SHA)).kind).toBe('error') + expect((await requestUploadURL('123', SHA)).kind).toBe('error') }) it('maps 200 upload responses to ok', async () => { stubFetch(200, {url: 'https://s3/put'}) - const result = await requestUploadURL('123') - expect(result).toEqual({kind: 'ok', url: 'https://s3/put'}) - }) - }) - - describe('mirror layout', () => { - it('keeps the mirror inside .git', () => { - expect(mirrorPath('/work/repo')).toBe( - path.join('/work/repo', '.git', 'wb-mirror.git') - ) - }) - - it('uses a relative alternates path that never leaves .git', () => { - expect(ALTERNATES_CONTENT).toBe('../wb-mirror.git/objects\n') - expect(path.isAbsolute(ALTERNATES_CONTENT.trim())).toBe(false) - }) - - it('writeAlternates writes the relative path into objects/info', async () => { - const workspace = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'wb-mirror-test-') - ) - try { - await writeAlternates(workspace) - const content = await fs.promises.readFile( - path.join(workspace, '.git', 'objects', 'info', 'alternates'), - 'utf8' - ) - expect(content).toBe(ALTERNATES_CONTENT) - } finally { - await fs.promises.rm(workspace, {recursive: true, force: true}) - } + expect(await requestUploadURL('123', SHA)).toEqual({ + kind: 'ok', + url: 'https://s3/put' + }) }) }) }) diff --git a/dist/index.js b/dist/index.js index 0882dfb..1cfe740 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41695,20 +41695,11 @@ function ref_helper_select(obj, path) { ;// CONCATENATED MODULE: external "stream/promises" const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts -/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +/* eslint-disable i18n-text/no-en -- upstream convention */ -// Thin client for backend-core's /api/v1/git-mirrors endpoints. -// -// Auth is the runner verification token every WarpBuild job carries in its env -// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the -// token alone. -// -// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): -// 200 -> use the presigned URL -// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) -// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror -// creation and upload entirely, behave exactly like stock actions/checkout -// else -> transient trouble: fall back WITHOUT the mirror download +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41716,9 +41707,9 @@ function backend_api_baseUrl() { function authHeader() { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; } -async function lookupMirror(repoKey) { +async function lookupSnapshot(repoKey, sha) { try { - const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, { + const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { headers: { authorization: authHeader() }, signal: AbortSignal.timeout(API_TIMEOUT_MS) }); @@ -41729,18 +41720,18 @@ async function lookupMirror(repoKey) { return { kind: 'miss' }; } if (res.status === 403) { - core_debug(`[wb-mirror] download-url answered 403 (disabled)`); + core_debug(`[wb-cache] download-url answered 403 (disabled)`); return { kind: 'disabled' }; } - core_debug(`[wb-mirror] download-url answered ${res.status}`); + core_debug(`[wb-cache] download-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-mirror] download-url failed: ${error}`); + core_debug(`[wb-cache] download-url failed: ${error}`); return { kind: 'error' }; } } -async function requestUploadURL(repoKey) { +async function requestUploadURL(repoKey, sha) { try { const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { method: 'POST', @@ -41748,7 +41739,7 @@ async function requestUploadURL(repoKey) { authorization: authHeader(), 'content-type': 'application/json' }, - body: JSON.stringify({ repo_key: repoKey }), + body: JSON.stringify({ repo_key: repoKey, sha }), signal: AbortSignal.timeout(API_TIMEOUT_MS) }); if (res.status === 200) { @@ -41759,22 +41750,20 @@ async function requestUploadURL(repoKey) { return { kind: 'error' }; } if (res.status === 403) { - core_debug(`[wb-mirror] upload-url answered 403 (disabled)`); + core_debug(`[wb-cache] upload-url answered 403 (disabled)`); return { kind: 'disabled' }; } - core_debug(`[wb-mirror] upload-url answered ${res.status}`); + core_debug(`[wb-cache] upload-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-mirror] upload-url failed: ${error}`); + core_debug(`[wb-cache] upload-url failed: ${error}`); return { kind: 'error' }; } } ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts -/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and - .js-suffixed ESM imports both follow upstream's own conventions; the import plugin - has no TS resolver configured in this repo. */ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ @@ -41783,48 +41772,27 @@ async function requestUploadURL(repoKey) { -// WarpBuild git-mirror cache. -// -// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we -// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at -// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and -// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers -// 404), THIS run creates the mirror: one full download of all branches + tags from -// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is -// disabled for this org (backend-driven kill switch) and we skip everything — no -// mirror creation, no upload. -// -// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path -// is relative, so it survives every container mount scheme: `container:` jobs (/__w), -// Docker container actions (/github/workspace), and `docker build COPY .`. -// -// Everything here is fail-open: any error or timeout degrades to stock actions/checkout -// behavior with a warning, never a failed checkout. -const MIRROR_DIR = 'wb-mirror.git'; -const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n`; +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const UPLOAD_TIMEOUT_MS = 15 * 60_000; -// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state -// everywhere outside WarpBuild); every other reason is logged at info. +// "hit " | "uploaded ", for e2e assertions. +const CACHE_STATE_FILE = 'wb-cache-state'; +// Logged at debug (normal state outside WarpBuild); other reasons log at info. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; -function mirrorPath(repositoryPath) { - return external_path_namespaceObject.join(repositoryPath, '.git', MIRROR_DIR); -} -// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache -// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and -// deliberately strict: anything unexpected means "behave exactly like upstream". +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; +let decision = 'off'; +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. function getMirrorCacheSkipReason(settings) { - // Only on WarpBuild runners (these are injected into every job's env there). if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { return SKIP_NOT_WARPBUILD; } - // Linux + macOS in v1. if (process.platform === 'win32') { - return 'Windows is not supported by the mirror cache yet'; + return 'Windows is not supported by the snapshot cache yet'; } - // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only - // cache when that is what is being checked out (`repository:` inputs fall back). const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set'; @@ -41833,181 +41801,183 @@ function getMirrorCacheSkipReason(settings) { if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; } - // github.com only (repo ids and mirror keys assume it). const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); if (server !== 'https://github.com') { return `server '${server}' is not github.com`; } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on'; + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`; + } + if (settings.fetchTags) { + return 'fetch-tags is enabled'; + } + if (settings.filter) { + return 'a fetch filter is configured'; + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured'; + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)'; + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref`; + } return null; } -// setup is the single upstream splice point, called right after `git init` + -// `git remote add` for a fresh repository. It never throws. -async function setup(settings, repositoryUrl) { +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +function computeDestinationRef(ref) { + if (!ref) { + return ''; + } + const upper = ref.toUpperCase(); + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`; + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`; + } + if (upper.startsWith('REFS/TAGS/')) { + return ref; + } + return null; +} +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +async function setup(settings) { + decision = 'off'; const skipReason = getMirrorCacheSkipReason(settings); if (skipReason) { if (skipReason === SKIP_NOT_WARPBUILD) { - core_debug(`WarpBuild mirror cache skipped: ${skipReason}`); + core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`); } else { - info(`WarpBuild mirror cache skipped: ${skipReason}`); + info(`WarpBuild snapshot cache skipped: ${skipReason}`); } - return; + return false; } - startGroup('WarpBuild: setting up git mirror cache'); + startGroup('WarpBuild: checkout snapshot cache'); try { - await setupInner(settings, repositoryUrl); + return await setupInner(settings); } catch (error) { - warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); + return false; } finally { endGroup(); } } -async function setupInner(settings, repositoryUrl) { +async function setupInner(settings) { const repoKey = process.env['GITHUB_REPOSITORY_ID']; - const mirror = mirrorPath(settings.repositoryPath); - // A second checkout of the same repo in one job finds the mirror already in place. - if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(mirror, 'objects'))) { - info('Mirror already present, reusing it'); - await writeAlternates(settings.repositoryPath); - return; - } - const lookup = await lookupMirror(repoKey); + const sha = settings.commit; + const lookup = await lookupSnapshot(repoKey, sha); if (lookup.kind === 'disabled') { - info('Mirror cache is disabled by the backend for this organization; using standard checkout'); - return; + info('Snapshot cache is disabled by the backend for this organization'); + return false; } if (lookup.kind === 'error') { - info('Mirror cache backend unavailable; using standard checkout'); - return; + info('Snapshot cache backend unavailable; using standard checkout'); + return false; } - if (lookup.kind === 'hit') { - info(`Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})`); - if (await restoreMirror(lookup.info.url, mirror)) { - await writeAlternates(settings.repositoryPath); - info('Mirror restored; the fetch below downloads only the delta'); - } - // Restore failure: fall through to plain checkout. The object exists, so the - // failure was transfer-shaped — re-downloading would just repeat the pain. - return; + if (lookup.kind === 'miss') { + info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); + decision = 'miss'; + return false; } - // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or - // unreachable backend never costs a wasted full mirror clone. - const probe = await requestUploadURL(repoKey); - if (probe.kind !== 'ok') { - info(probe.kind === 'disabled' - ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' - : 'Mirror cache backend unavailable; skipping mirror creation'); - return; + info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false; } - info('Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)'); - await createMirrorFromGitHub(settings, repositoryUrl, mirror); - await writeAlternates(settings.repositoryPath); - // Upload failures only warn: the local mirror still accelerates THIS run. - await uploadMirror(repoKey, mirror); + info('Snapshot restored; skipping the GitHub fetch entirely'); + return true; } -// writeAlternates points the workspace repo's object lookups at the mirror. The path is -// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. -async function writeAlternates(repositoryPath) { - const infoDir = external_path_namespaceObject.join(repositoryPath, '.git', 'objects', 'info'); - await external_fs_namespaceObject.promises.mkdir(infoDir, { recursive: true }); - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(infoDir, 'alternates'), ALTERNATES_CONTENT); - info(`Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}`); +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +async function contribute(settings) { + if (decision !== 'miss') { + return; + } + startGroup('WarpBuild: uploading checkout snapshot'); + try { + await uploadSnapshot(settings); + } + catch (error) { + warning(`Snapshot upload skipped: ${error}`); + } + finally { + endGroup(); + } } -async function restoreMirror(url, mirror) { - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-restore-${process.pid}.tar`); +async function restoreSnapshot(settings, url, sha) { + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); try { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }); if (!res.ok || !res.body) { - throw new Error(`mirror download answered ${res.status}`); + throw new Error(`snapshot download answered ${res.status}`); } await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); - await external_fs_namespaceObject.promises.mkdir(mirror, { recursive: true }); - await exec_exec('tar', ['-xf', tmpTar, '-C', mirror]); + await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); + const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`); + } + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref); + if (dstRef) { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]); + } + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`); return true; } catch (error) { - warning(`Mirror restore failed: ${error}`); - // Never leave a partial mirror behind an alternates file — that is corruption. - await external_fs_namespaceObject.promises.rm(mirror, { recursive: true, force: true }); + warning(`Snapshot restore failed: ${error}`); + // Reset .git to its freshly-init state. + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + recursive: true, + force: true + }); + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + recursive: true + }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + recursive: true + }); return false; } finally { await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); } } -// createMirrorFromGitHub builds the bare mirror by downloading the repository from -// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. -// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also -// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large -// share of the download on PR-heavy repos). PR-triggered checkouts still work: their -// head SHA simply arrives as a small delta in the workspace fetch. -// -// The full history download itself cannot be safely avoided: a shallow or partial -// mirror behind an alternates file is an incomplete object store that git assumes is -// complete — the exact corruption Blacksmith hit and reverted. A mirror must be -// complete with respect to the refs it advertises. -async function createMirrorFromGitHub(settings, repositoryUrl, mirror) { - // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars - // (git >= 2.31) so the credential never appears in any process's argv. - const basicCredential = Buffer.from(`x-access-token:${settings.authToken}`, 'utf8').toString('base64'); - core_setSecret(basicCredential); - const env = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined) { - env[key] = value; - } - } - env['GIT_CONFIG_COUNT'] = '1'; - env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader'; - env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}`; - await exec_exec('git', ['init', '--bare', '--quiet', mirror], { env }); - await exec_exec('git', ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], { - env - }); - // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. - await exec_exec('git', [ - '-c', - 'gc.auto=0', - '-C', - mirror, - 'fetch', - '--prune', - '--progress', - 'origin', - '+refs/heads/*:refs/heads/*', - '+refs/tags/*:refs/tags/*' - ], { env }); -} -async function uploadMirror(repoKey, mirror) { - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-mirror-upload-${process.pid}.tar`); +async function uploadSnapshot(settings) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const sha = settings.commit; + const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); + const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`); try { - // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic - // trims; the tar stays a valid bare repo either way. - await exec_exec('tar', [ - '-cf', - tmpTar, - '-C', - mirror, - '--exclude', - './hooks', - '--exclude', - './description', - '--exclude', - './FETCH_HEAD', - '.' - ]); + const members = ['objects']; + if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) { + members.push('shallow'); + } + await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]); const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; - // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the - // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. - const fresh = await requestUploadURL(repoKey); - if (fresh.kind !== 'ok') { - throw new Error(fresh.kind === 'disabled' - ? 'mirror cache was disabled by the backend' - : 'upload-url unavailable'); + const upload = await requestUploadURL(repoKey, sha); + if (upload.kind !== 'ok') { + info(upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading'); + return; } const init = { method: 'PUT', @@ -42019,14 +41989,12 @@ async function uploadMirror(repoKey, mirror) { duplex: 'half', signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) }; - const res = await fetch(fresh.url, init); + const res = await fetch(upload.url, init); if (!res.ok) { - throw new Error(`mirror upload answered ${res.status}`); + throw new Error(`snapshot upload answered ${res.status}`); } - info(`Mirror uploaded (${size} bytes); future runs will restore it`); - } - catch (error) { - warning(`Mirror upload skipped: ${error}`); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`); + info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`); } finally { await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); @@ -42066,6 +42034,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; + let warpbuildRestored = false; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -42116,9 +42085,8 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); - // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and - // point alternates at it so the fetch below downloads only the delta from GitHub. - await setup(settings, repositoryUrl); + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -42158,7 +42126,10 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + info('Skipping fetch: checkout was restored from the snapshot cache'); + } + else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); @@ -42224,6 +42195,8 @@ async function getSource(settings) { startGroup('Checking out the ref'); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); endGroup(); + // WarpBuild snapshot cache: upload the fetch result on a miss + await contribute(settings); // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index eedeb7f..3b5adbc 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -41,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null + let warpbuildRestored = false try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -132,9 +133,8 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.remoteAdd('origin', repositoryUrl) core.endGroup() - // WarpBuild git-mirror cache: restore (or hydrate) a bare mirror inside .git and - // point alternates at it so the fetch below downloads only the delta from GitHub. - await warpbuildMirror.setup(settings, repositoryUrl) + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildRestored = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -190,7 +190,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (settings.fetchDepth <= 0) { + if (warpbuildRestored) { + core.info('Skipping fetch: checkout was restored from the snapshot cache') + } else if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, @@ -276,6 +278,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) core.endGroup() + // WarpBuild snapshot cache: upload the fetch result on a miss + await warpbuildMirror.contribute(settings) + // Submodules if (settings.submodules) { // Temporarily override global config diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index 23eb8a1..d51cd06 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -1,29 +1,20 @@ -/* eslint-disable i18n-text/no-en -- log/error strings are English by upstream convention */ +/* eslint-disable i18n-text/no-en -- upstream convention */ import * as core from '@actions/core' -// Thin client for backend-core's /api/v1/git-mirrors endpoints. -// -// Auth is the runner verification token every WarpBuild job carries in its env -// (WARPBUILD_RUNNER_VERIFICATION_TOKEN); the backend resolves instance → org from the -// token alone. -// -// HTTP contract (mirrors internal/runners/internal/git_mirror_service.go): -// 200 -> use the presigned URL -// 404 -> cache miss: create the mirror (download from GitHub + tar + upload) -// 403 -> feature disabled for this org (backend-driven kill switch): skip mirror -// creation and upload entirely, behave exactly like stock actions/checkout -// else -> transient trouble: fall back WITHOUT the mirror download +// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner +// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the +// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. const API_TIMEOUT_MS = 10_000 -export interface MirrorDownloadInfo { +export interface SnapshotDownloadInfo { url: string size_bytes: number created_at: string } -export type MirrorLookup = - | {kind: 'hit'; info: MirrorDownloadInfo} +export type SnapshotLookup = + | {kind: 'hit'; info: SnapshotDownloadInfo} | {kind: 'miss'} | {kind: 'disabled'} | {kind: 'error'} @@ -41,35 +32,41 @@ function authHeader(): string { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` } -export async function lookupMirror(repoKey: string): Promise { +export async function lookupSnapshot( + repoKey: string, + sha: string +): Promise { try { const res = await fetch( - `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}`, + `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( + repoKey + )}&sha=${encodeURIComponent(sha)}`, { headers: {authorization: authHeader()}, signal: AbortSignal.timeout(API_TIMEOUT_MS) } ) if (res.status === 200) { - return {kind: 'hit', info: (await res.json()) as MirrorDownloadInfo} + return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} } if (res.status === 404) { return {kind: 'miss'} } if (res.status === 403) { - core.debug(`[wb-mirror] download-url answered 403 (disabled)`) + core.debug(`[wb-cache] download-url answered 403 (disabled)`) return {kind: 'disabled'} } - core.debug(`[wb-mirror] download-url answered ${res.status}`) + core.debug(`[wb-cache] download-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-mirror] download-url failed: ${error}`) + core.debug(`[wb-cache] download-url failed: ${error}`) return {kind: 'error'} } } export async function requestUploadURL( - repoKey: string + repoKey: string, + sha: string ): Promise { try { const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { @@ -78,7 +75,7 @@ export async function requestUploadURL( authorization: authHeader(), 'content-type': 'application/json' }, - body: JSON.stringify({repo_key: repoKey}), + body: JSON.stringify({repo_key: repoKey, sha}), signal: AbortSignal.timeout(API_TIMEOUT_MS) }) if (res.status === 200) { @@ -89,13 +86,13 @@ export async function requestUploadURL( return {kind: 'error'} } if (res.status === 403) { - core.debug(`[wb-mirror] upload-url answered 403 (disabled)`) + core.debug(`[wb-cache] upload-url answered 403 (disabled)`) return {kind: 'disabled'} } - core.debug(`[wb-mirror] upload-url answered ${res.status}`) + core.debug(`[wb-cache] upload-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-mirror] upload-url failed: ${error}`) + core.debug(`[wb-cache] upload-url failed: ${error}`) return {kind: 'error'} } } diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 03f5f91..9e604e1 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -1,6 +1,4 @@ -/* eslint-disable i18n-text/no-en, import/no-unresolved -- English log strings and - .js-suffixed ESM imports both follow upstream's own conventions; the import plugin - has no TS resolver configured in this repo. */ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as fs from 'fs' @@ -11,58 +9,38 @@ import {pipeline} from 'stream/promises' import {IGitSourceSettings} from '../git-source-settings.js' import * as api from './backend-api.js' -// WarpBuild git-mirror cache. -// -// A tar of the repo's bare mirror lives in a WarpBuild-owned S3 bucket. At checkout we -// restore it to /.git/wb-mirror.git and point .git/objects/info/alternates at -// it, so the (unmodified) upstream fetch advertises the mirror's ref tips as "haves" and -// GitHub only sends objects the mirror doesn't already hold. On a miss (backend answers -// 404), THIS run creates the mirror: one full download of all branches + tags from -// GitHub, then tar + presigned PUT. When the backend answers 403 the feature is -// disabled for this org (backend-driven kill switch) and we skip everything — no -// mirror creation, no upload. -// -// The mirror lives INSIDE .git (same precedent as .git/modules) and the alternates path -// is relative, so it survives every container mount scheme: `container:` jobs (/__w), -// Docker container actions (/github/workspace), and `docker build COPY .`. -// -// Everything here is fail-open: any error or timeout degrades to stock actions/checkout -// behavior with a warning, never a failed checkout. - -const MIRROR_DIR = 'wb-mirror.git' -export const ALTERNATES_CONTENT = `../${MIRROR_DIR}/objects\n` +// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch +// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are +// immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const UPLOAD_TIMEOUT_MS = 15 * 60_000 -// Skip reason for "not a WarpBuild runner" — logged at debug (it is the normal state -// everywhere outside WarpBuild); every other reason is logged at info. +// "hit " | "uploaded ", for e2e assertions. +export const CACHE_STATE_FILE = 'wb-cache-state' + +// Logged at debug (normal state outside WarpBuild); other reasons log at info. export const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)' -export function mirrorPath(repositoryPath: string): string { - return path.join(repositoryPath, '.git', MIRROR_DIR) -} +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ -// getMirrorCacheSkipReason gates the whole feature. Returns null when the mirror cache -// should be attempted, otherwise a human-readable reason to log. Cheap, pure, and -// deliberately strict: anything unexpected means "behave exactly like upstream". +let decision: 'off' | 'miss' = 'off' + +// Null = attempt the cache; else a reason to log. Only the default checkout shape +// is served. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { - // Only on WarpBuild runners (these are injected into every job's env there). if ( !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL'] ) { return SKIP_NOT_WARPBUILD } - // Linux + macOS in v1. if (process.platform === 'win32') { - return 'Windows is not supported by the mirror cache yet' + return 'Windows is not supported by the snapshot cache yet' } - // The cache key is GITHUB_REPOSITORY_ID, which belongs to the WORKFLOW repo — so only - // cache when that is what is being checked out (`repository:` inputs fall back). const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set' @@ -71,7 +49,6 @@ export function getMirrorCacheSkipReason( if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` } - // github.com only (repo ids and mirror keys assume it). const server = (settings.githubServerUrl || 'https://github.com').replace( /\/+$/, '' @@ -79,226 +56,208 @@ export function getMirrorCacheSkipReason( if (server !== 'https://github.com') { return `server '${server}' is not github.com` } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on' + } + if (settings.fetchDepth !== 1) { + return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1` + } + if (settings.fetchTags) { + return 'fetch-tags is enabled' + } + if (settings.filter) { + return 'a fetch filter is configured' + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured' + } + if (settings.lfs) { + return 'lfs is enabled (lfs objects are not in the snapshot)' + } + if (settings.ref && computeDestinationRef(settings.ref) === null) { + return `ref '${settings.ref}' has no cacheable destination ref` + } return null } -// setup is the single upstream splice point, called right after `git init` + -// `git remote add` for a fresh repository. It never throws. -export async function setup( - settings: IGitSourceSettings, - repositoryUrl: string -): Promise { +// The local ref the fetch would have created ('' = none needed, null = uncacheable). +export function computeDestinationRef(ref: string): string | null { + if (!ref) { + return '' + } + const upper = ref.toUpperCase() + if (upper.startsWith('REFS/HEADS/')) { + return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}` + } + if (upper.startsWith('REFS/PULL/')) { + return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}` + } + if (upper.startsWith('REFS/TAGS/')) { + return ref + } + return null +} + +// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +export async function setup(settings: IGitSourceSettings): Promise { + decision = 'off' const skipReason = getMirrorCacheSkipReason(settings) if (skipReason) { if (skipReason === SKIP_NOT_WARPBUILD) { - core.debug(`WarpBuild mirror cache skipped: ${skipReason}`) + core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`) } else { - core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) } - return + return false } - core.startGroup('WarpBuild: setting up git mirror cache') + core.startGroup('WarpBuild: checkout snapshot cache') try { - await setupInner(settings, repositoryUrl) + return await setupInner(settings) } catch (error) { core.warning( - `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + `WarpBuild snapshot cache unavailable, using standard checkout: ${error}` ) + return false } finally { core.endGroup() } } -async function setupInner( - settings: IGitSourceSettings, - repositoryUrl: string -): Promise { +async function setupInner(settings: IGitSourceSettings): Promise { const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string - const mirror = mirrorPath(settings.repositoryPath) + const sha = settings.commit - // A second checkout of the same repo in one job finds the mirror already in place. - if (fs.existsSync(path.join(mirror, 'objects'))) { - core.info('Mirror already present, reusing it') - await writeAlternates(settings.repositoryPath) - return - } - - const lookup = await api.lookupMirror(repoKey) + const lookup = await api.lookupSnapshot(repoKey, sha) if (lookup.kind === 'disabled') { - core.info( - 'Mirror cache is disabled by the backend for this organization; using standard checkout' - ) - return + core.info('Snapshot cache is disabled by the backend for this organization') + return false } - if (lookup.kind === 'error') { - core.info('Mirror cache backend unavailable; using standard checkout') - return + core.info('Snapshot cache backend unavailable; using standard checkout') + return false } - if (lookup.kind === 'hit') { + if (lookup.kind === 'miss') { core.info( - `Cache hit: restoring mirror (${lookup.info.size_bytes} bytes, created ${lookup.info.created_at})` + `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` ) - if (await restoreMirror(lookup.info.url, mirror)) { - await writeAlternates(settings.repositoryPath) - core.info('Mirror restored; the fetch below downloads only the delta') - } - return - } - - // Miss. Probe upload authorization BEFORE the expensive clone so a disabled or - // unreachable backend never costs a wasted full mirror clone. - const probe = await api.requestUploadURL(repoKey) - if (probe.kind !== 'ok') { - core.info( - probe.kind === 'disabled' - ? 'Mirror cache is disabled by the backend for this organization; using standard checkout' - : 'Mirror cache backend unavailable; skipping mirror creation' - ) - return + decision = 'miss' + return false } core.info( - 'Cache miss: downloading all branches and tags from GitHub into a fresh mirror (one-time; later runs download only the delta)' + `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` ) - await createMirrorFromGitHub(settings, repositoryUrl, mirror) - await writeAlternates(settings.repositoryPath) - // Upload failures only warn: the local mirror still accelerates THIS run. - await uploadMirror(repoKey, mirror) + if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { + return false + } + core.info('Snapshot restored; skipping the GitHub fetch entirely') + return true } -// writeAlternates points the workspace repo's object lookups at the mirror. The path is -// RELATIVE (resolved against .git/objects), which is what makes container remaps safe. -export async function writeAlternates(repositoryPath: string): Promise { - const infoDir = path.join(repositoryPath, '.git', 'objects', 'info') - await fs.promises.mkdir(infoDir, {recursive: true}) - await fs.promises.writeFile( - path.join(infoDir, 'alternates'), - ALTERNATES_CONTENT - ) - core.info( - `Wrote .git/objects/info/alternates -> ${ALTERNATES_CONTENT.trim()}` - ) +// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +export async function contribute(settings: IGitSourceSettings): Promise { + if (decision !== 'miss') { + return + } + core.startGroup('WarpBuild: uploading checkout snapshot') + try { + await uploadSnapshot(settings) + } catch (error) { + core.warning(`Snapshot upload skipped: ${error}`) + } finally { + core.endGroup() + } } -async function restoreMirror(url: string, mirror: string): Promise { - const tmpTar = path.join(os.tmpdir(), `wb-mirror-restore-${process.pid}.tar`) +async function restoreSnapshot( + settings: IGitSourceSettings, + url: string, + sha: string +): Promise { + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`) try { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }) if (!res.ok || !res.body) { - throw new Error(`mirror download answered ${res.status}`) + throw new Error(`snapshot download answered ${res.status}`) } await pipeline( Readable.fromWeb(res.body as import('stream/web').ReadableStream), fs.createWriteStream(tmpTar) ) - await fs.promises.mkdir(mirror, {recursive: true}) - await exec.exec('tar', ['-xf', tmpTar, '-C', mirror]) + await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) + + const check = await exec.exec( + 'git', + ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], + {ignoreReturnCode: true} + ) + if (check !== 0) { + throw new Error(`restored snapshot does not contain ${sha}`) + } + + // The ref the skipped fetch would have created; upstream's verification needs it. + const dstRef = computeDestinationRef(settings.ref) + if (dstRef) { + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'update-ref', + dstRef, + sha + ]) + } + + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `hit ${sha}\n` + ) return true } catch (error) { - core.warning(`Mirror restore failed: ${error}`) - // Never leave a partial mirror behind an alternates file — that is corruption. - await fs.promises.rm(mirror, {recursive: true, force: true}) + core.warning(`Snapshot restore failed: ${error}`) + // Reset .git to its freshly-init state. + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }) + await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) return false } finally { await fs.promises.rm(tmpTar, {force: true}) } } -// createMirrorFromGitHub builds the bare mirror by downloading the repository from -// GitHub — the one heavy operation in the whole design, paid once per repo per TTL. -// Scope is deliberately branches + tags only (NOT `clone --mirror`, which would also -// copy refs/pull/* — every PR head ever opened, unbounded growth and often a large -// share of the download on PR-heavy repos). PR-triggered checkouts still work: their -// head SHA simply arrives as a small delta in the workspace fetch. -// -// The full history download itself cannot be safely avoided: a shallow or partial -// mirror behind an alternates file is an incomplete object store that git assumes is -// complete — the exact corruption Blacksmith hit and reverted. A mirror must be -// complete with respect to the refs it advertises. -async function createMirrorFromGitHub( - settings: IGitSourceSettings, - repositoryUrl: string, - mirror: string -): Promise { - // Same header shape as upstream auth, but passed via GIT_CONFIG_* env vars - // (git >= 2.31) so the credential never appears in any process's argv. - const basicCredential = Buffer.from( - `x-access-token:${settings.authToken}`, - 'utf8' - ).toString('base64') - core.setSecret(basicCredential) - - const env: {[key: string]: string} = {} - for (const [key, value] of Object.entries(process.env)) { - if (value !== undefined) { - env[key] = value - } - } - env['GIT_CONFIG_COUNT'] = '1' - env['GIT_CONFIG_KEY_0'] = 'http.https://github.com/.extraheader' - env['GIT_CONFIG_VALUE_0'] = `AUTHORIZATION: basic ${basicCredential}` - - await exec.exec('git', ['init', '--bare', '--quiet', mirror], {env}) - await exec.exec( - 'git', - ['-C', mirror, 'remote', 'add', 'origin', repositoryUrl], - { - env - } - ) - // gc.auto=0: never let the fetch spawn a detached gc that outlives the step. - await exec.exec( - 'git', - [ - '-c', - 'gc.auto=0', - '-C', - mirror, - 'fetch', - '--prune', - '--progress', - 'origin', - '+refs/heads/*:refs/heads/*', - '+refs/tags/*:refs/tags/*' - ], - {env} - ) -} - -async function uploadMirror(repoKey: string, mirror: string): Promise { - const tmpTar = path.join(os.tmpdir(), `wb-mirror-upload-${process.pid}.tar`) +async function uploadSnapshot(settings: IGitSourceSettings): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const sha = settings.commit + const gitDir = path.join(settings.repositoryPath, '.git') + const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`) try { - // Plain tar, no gzip (pack data is already zlib-compressed). Excludes are cosmetic - // trims; the tar stays a valid bare repo either way. - await exec.exec('tar', [ - '-cf', - tmpTar, - '-C', - mirror, - '--exclude', - './hooks', - '--exclude', - './description', - '--exclude', - './FETCH_HEAD', - '.' - ]) + const members = ['objects'] + if (fs.existsSync(path.join(gitDir, 'shallow'))) { + members.push('shallow') + } + await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]) const size = (await fs.promises.stat(tmpTar)).size - // Fresh URL after the potentially long clone+tar (presigned PUTs expire). If the - // backend flipped to disabled meanwhile, this answers 403 and the upload is skipped. - const fresh = await api.requestUploadURL(repoKey) - if (fresh.kind !== 'ok') { - throw new Error( - fresh.kind === 'disabled' - ? 'mirror cache was disabled by the backend' - : 'upload-url unavailable' + const upload = await api.requestUploadURL(repoKey, sha) + if (upload.kind !== 'ok') { + core.info( + upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading' ) + return } const init: RequestInit & {duplex: 'half'} = { @@ -313,13 +272,17 @@ async function uploadMirror(repoKey: string, mirror: string): Promise { duplex: 'half', signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) } - const res = await fetch(fresh.url, init) + const res = await fetch(upload.url, init) if (!res.ok) { - throw new Error(`mirror upload answered ${res.status}`) + throw new Error(`snapshot upload answered ${res.status}`) } - core.info(`Mirror uploaded (${size} bytes); future runs will restore it`) - } catch (error) { - core.warning(`Mirror upload skipped: ${error}`) + await fs.promises.writeFile( + path.join(gitDir, CACHE_STATE_FILE), + `uploaded ${sha}\n` + ) + core.info( + `Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch` + ) } finally { await fs.promises.rm(tmpTar, {force: true}) } From b3ad80abe3e79976564272ca5534ddfd0e0b7536 Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 15:29:01 +0530 Subject: [PATCH 03/13] add lock awareness --- README.md | 12 ++--- __test__/warpbuild-mirror.test.ts | 73 +++++++++++++++++++++++++++++++ dist/index.js | 38 ++++++++++++++-- src/warpbuild/backend-api.ts | 8 +++- src/warpbuild/mirror-cache.ts | 34 ++++++++++++-- 5 files changed, 152 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e2e3c27..89eea5d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # WarpBuild Checkout This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in -replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's -bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git -alternates, so the fetch from GitHub downloads only the delta instead of the whole -repository. On a cache miss, the run hydrates the mirror once (full clone + upload); -mirrors expire on a server-configured TTL and re-hydrate. +replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` +objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA. +A later job checking out the same commit restores that snapshot and skips the fetch from +GitHub entirely — cutting request load (the main source of GitHub rate-limiting on +matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. +- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS); + everything else runs exactly like upstream. - Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. - Fail-open — any cache error degrades to stock `actions/checkout` behavior. - All fork code lives in `src/warpbuild/` diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 1f75b01..cbaa272 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,8 +6,13 @@ import { afterEach, afterAll } from '@jest/globals' +import * as exec from '@actions/exec' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' import { SKIP_NOT_WARPBUILD, + assertSafeTarMembers, computeDestinationRef, getMirrorCacheSkipReason } from '../src/warpbuild/mirror-cache.js' @@ -131,6 +136,69 @@ describe('warpbuild snapshot cache', () => { }) }) + describe('assertSafeTarMembers', () => { + async function makeTar( + dir: string, + members: string[], + writeFiles = true + ): Promise { + if (writeFiles) { + for (const m of members) { + const abs = path.join(dir, m) + await fs.promises.mkdir(path.dirname(abs), {recursive: true}) + await fs.promises.writeFile(abs, 'x') + } + } + const tar = path.join(dir, 'out.tar') + await exec.exec('tar', ['-cf', tar, '-C', dir, ...members]) + return tar + } + + it('accepts objects/ and shallow members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow']) + await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined() + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects members outside objects/ and shallow (e.g. hooks)', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit']) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + + it('rejects path traversal members', async () => { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) + try { + await fs.promises.mkdir(path.join(dir, 'sub')) + const tar = path.join(dir, 'evil.tar') + // -P preserves the ../ member instead of stripping it. + await fs.promises.writeFile(path.join(dir, 'evil'), 'x') + await exec.exec('tar', [ + '-cPf', + tar, + '-C', + path.join(dir, 'sub'), + '../evil' + ]) + await expect(assertSafeTarMembers(tar)).rejects.toThrow( + /unexpected snapshot tar member/ + ) + } finally { + await fs.promises.rm(dir, {recursive: true, force: true}) + } + }) + }) + describe('computeDestinationRef', () => { it('maps the refs the fetch would have created', () => { expect(computeDestinationRef('refs/heads/main')).toBe( @@ -172,6 +240,11 @@ describe('warpbuild snapshot cache', () => { expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') }) + it('maps 409 upload responses to locked (another job is uploading)', async () => { + stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) + expect((await requestUploadURL('123', SHA)).kind).toBe('locked') + }) + it('maps 403 to disabled (backend kill switch)', async () => { stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') diff --git a/dist/index.js b/dist/index.js index 1cfe740..c91eee1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41699,7 +41699,8 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. +// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is +// uploading this snapshot (skip upload); else = fall back. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41753,6 +41754,10 @@ async function requestUploadURL(repoKey, sha) { core_debug(`[wb-cache] upload-url answered 403 (disabled)`); return { kind: 'disabled' }; } + if (res.status === 409) { + core_debug(`[wb-cache] upload-url answered 409 (locked)`); + return { kind: 'locked' }; + } core_debug(`[wb-cache] upload-url answered ${res.status}`); return { kind: 'error' }; } @@ -41910,6 +41915,28 @@ async function contribute(settings) { endGroup(); } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +async function assertSafeTarMembers(tar) { + let listing = ''; + await exec_exec('tar', ['-tf', tar], { + silent: true, + listeners: { stdout: (d) => (listing += d.toString()) } + }); + for (const raw of listing.split('\n')) { + const member = raw.trim(); + if (!member) { + continue; + } + const top = member.replace(/^\.\//, '').split('/')[0]; + if (member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow')) { + throw new Error(`unexpected snapshot tar member: ${member}`); + } + } +} async function restoreSnapshot(settings, url, sha) { const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); @@ -41921,6 +41948,7 @@ async function restoreSnapshot(settings, url, sha) { throw new Error(`snapshot download answered ${res.status}`); } await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); + await assertSafeTarMembers(tmpTar); await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); if (check !== 0) { @@ -41974,9 +42002,11 @@ async function uploadSnapshot(settings) { const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; const upload = await requestUploadURL(repoKey, sha); if (upload.kind !== 'ok') { - info(upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading'); + info(upload.kind === 'locked' + ? 'Another job is already uploading this snapshot; skipping' + : upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading'); return; } const init = { diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index d51cd06..5f6ca85 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -3,7 +3,8 @@ import * as core from '@actions/core' // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); else = fall back. +// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is +// uploading this snapshot (skip upload); else = fall back. const API_TIMEOUT_MS = 10_000 @@ -22,6 +23,7 @@ export type SnapshotLookup = export type UploadURLResult = | {kind: 'ok'; url: string} | {kind: 'disabled'} + | {kind: 'locked'} | {kind: 'error'} function baseUrl(): string { @@ -89,6 +91,10 @@ export async function requestUploadURL( core.debug(`[wb-cache] upload-url answered 403 (disabled)`) return {kind: 'disabled'} } + if (res.status === 409) { + core.debug(`[wb-cache] upload-url answered 409 (locked)`) + return {kind: 'locked'} + } core.debug(`[wb-cache] upload-url answered ${res.status}`) return {kind: 'error'} } catch (error) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 9e604e1..dbeb476 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -171,6 +171,31 @@ export async function contribute(settings: IGitSourceSettings): Promise { } } +// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` +// component — the tar is remote and extracted into .git, so a crafted member could +// escape (e.g. hooks/, ../) and run code during checkout. +export async function assertSafeTarMembers(tar: string): Promise { + let listing = '' + await exec.exec('tar', ['-tf', tar], { + silent: true, + listeners: {stdout: (d: Buffer) => (listing += d.toString())} + }) + for (const raw of listing.split('\n')) { + const member = raw.trim() + if (!member) { + continue + } + const top = member.replace(/^\.\//, '').split('/')[0] + if ( + member.startsWith('/') || + member.split('/').includes('..') || + (top !== 'objects' && top !== 'shallow') + ) { + throw new Error(`unexpected snapshot tar member: ${member}`) + } + } +} + async function restoreSnapshot( settings: IGitSourceSettings, url: string, @@ -189,6 +214,7 @@ async function restoreSnapshot( Readable.fromWeb(res.body as import('stream/web').ReadableStream), fs.createWriteStream(tmpTar) ) + await assertSafeTarMembers(tmpTar) await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) const check = await exec.exec( @@ -253,9 +279,11 @@ async function uploadSnapshot(settings: IGitSourceSettings): Promise { const upload = await api.requestUploadURL(repoKey, sha) if (upload.kind !== 'ok') { core.info( - upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading' + upload.kind === 'locked' + ? 'Another job is already uploading this snapshot; skipping' + : upload.kind === 'disabled' + ? 'Snapshot cache is disabled by the backend; not uploading' + : 'Snapshot cache backend unavailable; not uploading' ) return } From 8b5fffd5f306c1d02b744fdcba8b4d11e697ae2a Mon Sep 17 00:00:00 2001 From: darshanime Date: Wed, 8 Jul 2026 16:51:09 +0530 Subject: [PATCH 04/13] ungate windows --- dist/index.js | 9 ++++++--- src/warpbuild/mirror-cache.ts | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dist/index.js b/dist/index.js index c91eee1..c090b4a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41777,6 +41777,7 @@ async function requestUploadURL(repoKey, sha) { + // WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch // produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are // immutable (no expiry). Fail-open: any error degrades to stock behavior. @@ -41795,9 +41796,6 @@ function getMirrorCacheSkipReason(settings) { !process.env['WARPBUILD_HOST_URL']) { return SKIP_NOT_WARPBUILD; } - if (process.platform === 'win32') { - return 'Windows is not supported by the snapshot cache yet'; - } const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set'; @@ -41878,6 +41876,11 @@ async function setup(settings) { async function setupInner(settings) { const repoKey = process.env['GITHUB_REPOSITORY_ID']; const sha = settings.commit; + // The restore/upload paths shell out to `tar`; without it, fall back cleanly. + if (!(await which('tar', false))) { + info('tar not found on PATH; using standard checkout'); + return false; + } const lookup = await lookupSnapshot(repoKey, sha); if (lookup.kind === 'disabled') { info('Snapshot cache is disabled by the backend for this organization'); diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index dbeb476..879a43f 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -1,6 +1,7 @@ /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ import * as core from '@actions/core' import * as exec from '@actions/exec' +import * as io from '@actions/io' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' @@ -38,9 +39,6 @@ export function getMirrorCacheSkipReason( ) { return SKIP_NOT_WARPBUILD } - if (process.platform === 'win32') { - return 'Windows is not supported by the snapshot cache yet' - } const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' if (!repoKey) { return 'GITHUB_REPOSITORY_ID is not set' @@ -127,6 +125,12 @@ async function setupInner(settings: IGitSourceSettings): Promise { const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string const sha = settings.commit + // The restore/upload paths shell out to `tar`; without it, fall back cleanly. + if (!(await io.which('tar', false))) { + core.info('tar not found on PATH; using standard checkout') + return false + } + const lookup = await api.lookupSnapshot(repoKey, sha) if (lookup.kind === 'disabled') { From 41e531959617eba7e5004d32d15b483ad73fdf18 Mon Sep 17 00:00:00 2001 From: darshanime Date: Thu, 9 Jul 2026 14:07:20 +0530 Subject: [PATCH 05/13] migrate to post job uploads --- README.md | 2 +- __test__/warpbuild-mirror.test.ts | 91 ++++++++++++++---- action.yml | 2 + dist/index.js | 154 +++++++++--------------------- src/warpbuild/backend-api.ts | 50 +--------- src/warpbuild/mirror-cache.ts | 137 +++++++++++--------------- 6 files changed, 183 insertions(+), 253 deletions(-) diff --git a/README.md b/README.md index 89eea5d..d12db7f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` -objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA. +objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA. A later job checking out the same commit restores that snapshot and skips the fetch from GitHub entirely — cutting request load (the main source of GitHub rate-limiting on matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index cbaa272..7d280fd 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -14,9 +14,11 @@ import { SKIP_NOT_WARPBUILD, assertSafeTarMembers, computeDestinationRef, - getMirrorCacheSkipReason + getMirrorCacheSkipReason, + manifestDir, + resetGitObjects } from '../src/warpbuild/mirror-cache.js' -import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js' +import {lookupSnapshot} from '../src/warpbuild/backend-api.js' import {IGitSourceSettings} from '../src/git-source-settings.js' const WB_ENV = [ @@ -136,6 +138,60 @@ describe('warpbuild snapshot cache', () => { }) }) + describe('resetGitObjects', () => { + it('restores the empty git-init object layout, dropping restored junk', async () => { + const gitDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-reset-') + ) + try { + // Simulate a partially-restored .git: stray objects, a pack, a shallow file. + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) + await fs.promises.writeFile( + path.join(gitDir, 'objects', 'ab'), + 'junk loose object' + ) + await fs.promises.writeFile( + path.join(gitDir, 'objects', 'pack', 'pack-x.pack'), + 'junk pack' + ) + await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n') + + await resetGitObjects(gitDir) + + // shallow gone; objects/ holds only the two empty init dirs. + expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false) + expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false) + expect( + (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() + ).toEqual(['info', 'pack']) + expect( + await fs.promises.readdir(path.join(gitDir, 'objects', 'pack')) + ).toEqual([]) + expect( + await fs.promises.readdir(path.join(gitDir, 'objects', 'info')) + ).toEqual([]) + } finally { + await fs.promises.rm(gitDir, {recursive: true, force: true}) + } + }) + + it('is a no-op-safe on an already-clean .git', async () => { + const gitDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-reset-') + ) + try { + await expect(resetGitObjects(gitDir)).resolves.toBeUndefined() + expect( + (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() + ).toEqual(['info', 'pack']) + } finally { + await fs.promises.rm(gitDir, {recursive: true, force: true}) + } + }) + }) + describe('assertSafeTarMembers', () => { async function makeTar( dir: string, @@ -235,20 +291,14 @@ describe('warpbuild snapshot cache', () => { expect(lastUrl).toContain(`sha=${SHA}`) }) - it('maps 404 to miss (upload after the stock fetch)', async () => { + it('maps 404 to miss (agent uploads after the job)', async () => { stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') }) - it('maps 409 upload responses to locked (another job is uploading)', async () => { - stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) - expect((await requestUploadURL('123', SHA)).kind).toBe('locked') - }) - it('maps 403 to disabled (backend kill switch)', async () => { stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') - expect((await requestUploadURL('123', SHA)).kind).toBe('disabled') }) it('maps other statuses and network failures to error', async () => { @@ -258,15 +308,24 @@ describe('warpbuild snapshot cache', () => { throw new Error('boom') }) as typeof fetch expect((await lookupSnapshot('123', SHA)).kind).toBe('error') - expect((await requestUploadURL('123', SHA)).kind).toBe('error') + }) + }) + + describe('manifestDir', () => { + const saved = process.env['RUNNER_TEMP'] + afterEach(() => { + if (saved === undefined) { + delete process.env['RUNNER_TEMP'] + } else { + process.env['RUNNER_TEMP'] = saved + } }) - it('maps 200 upload responses to ok', async () => { - stubFetch(200, {url: 'https://s3/put'}) - expect(await requestUploadURL('123', SHA)).toEqual({ - kind: 'ok', - url: 'https://s3/put' - }) + it('places manifests under RUNNER_TEMP/wb-snapshots', () => { + process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x') + expect(manifestDir()).toBe( + path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots') + ) }) }) }) diff --git a/action.yml b/action.yml index 5b0524f..32c44db 100644 --- a/action.yml +++ b/action.yml @@ -112,6 +112,8 @@ outputs: description: 'The branch, tag or SHA that was checked out' commit: description: 'The commit SHA that was checked out' + cache-hit: + description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)' runs: using: node24 main: dist/index.js diff --git a/dist/index.js b/dist/index.js index c090b4a..0a42bda 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41697,10 +41697,9 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts /* eslint-disable i18n-text/no-en -- upstream convention */ -// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner -// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is -// uploading this snapshot (skip upload); else = fall back. +// Client for backend-core's git-mirror download-url endpoint, authed by the runner +// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable +// org; else = fall back. The upload half runs in the warpbuild-agent after the job. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41732,40 +41731,6 @@ async function lookupSnapshot(repoKey, sha) { return { kind: 'error' }; } } -async function requestUploadURL(repoKey, sha) { - try { - const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, { - method: 'POST', - headers: { - authorization: authHeader(), - 'content-type': 'application/json' - }, - body: JSON.stringify({ repo_key: repoKey, sha }), - signal: AbortSignal.timeout(API_TIMEOUT_MS) - }); - if (res.status === 200) { - const body = (await res.json()); - if (body.url) { - return { kind: 'ok', url: body.url }; - } - return { kind: 'error' }; - } - if (res.status === 403) { - core_debug(`[wb-cache] upload-url answered 403 (disabled)`); - return { kind: 'disabled' }; - } - if (res.status === 409) { - core_debug(`[wb-cache] upload-url answered 409 (locked)`); - return { kind: 'locked' }; - } - core_debug(`[wb-cache] upload-url answered ${res.status}`); - return { kind: 'error' }; - } - catch (error) { - core_debug(`[wb-cache] upload-url failed: ${error}`); - return { kind: 'error' }; - } -} ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ @@ -41782,9 +41747,6 @@ async function requestUploadURL(repoKey, sha) { // produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are // immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; -const UPLOAD_TIMEOUT_MS = 15 * 60_000; -// "hit " | "uploaded ", for e2e assertions. -const CACHE_STATE_FILE = 'wb-cache-state'; // Logged at debug (normal state outside WarpBuild); other reasons log at info. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; @@ -41849,16 +41811,17 @@ function computeDestinationRef(ref) { return null; } // Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +// Sets the `cache-hit` action output (true only on a restore). async function setup(settings) { + const restored = await setupImpl(settings); + setOutput('cache-hit', restored ? 'true' : 'false'); + return restored; +} +async function setupImpl(settings) { decision = 'off'; const skipReason = getMirrorCacheSkipReason(settings); if (skipReason) { - if (skipReason === SKIP_NOT_WARPBUILD) { - core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`); - } - else { - info(`WarpBuild snapshot cache skipped: ${skipReason}`); - } + info(`WarpBuild snapshot cache skipped: ${skipReason}`); return false; } startGroup('WarpBuild: checkout snapshot cache'); @@ -41902,20 +41865,17 @@ async function setupInner(settings) { info('Snapshot restored; skipping the GitHub fetch entirely'); return true; } -// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the +// snapshot after the job exits (keeping tar+upload off the customer's billed time). async function contribute(settings) { if (decision !== 'miss') { return; } - startGroup('WarpBuild: uploading checkout snapshot'); try { - await uploadSnapshot(settings); + await writeSnapshotManifest(settings); } catch (error) { - warning(`Snapshot upload skipped: ${error}`); - } - finally { - endGroup(); + warning(`Snapshot manifest not written: ${error}`); } } // Refuse any tar member outside objects/ or shallow, absolute, or with a `..` @@ -41968,70 +41928,50 @@ async function restoreSnapshot(settings, url, sha) { sha ]); } - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`); return true; } catch (error) { warning(`Snapshot restore failed: ${error}`); - // Reset .git to its freshly-init state. - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { - recursive: true, - force: true - }); - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { - recursive: true - }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { - recursive: true - }); + await resetGitObjects(gitDir); return false; } finally { await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); } } -async function uploadSnapshot(settings) { - const repoKey = process.env['GITHUB_REPOSITORY_ID']; +// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` +// creates, so a failed restore leaves no trace for the fallback fetch. +async function resetGitObjects(gitDir) { + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + recursive: true, + force: true + }); + await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + recursive: true + }); + await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + recursive: true + }); +} +// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which +// is /_work/_temp — host-visible to the agent (it knows the runner dir). +function manifestDir() { + const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir(); + return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots'); +} +// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. +async function writeSnapshotManifest(settings) { const sha = settings.commit; - const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`); - try { - const members = ['objects']; - if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) { - members.push('shallow'); - } - await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]); - const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; - const upload = await requestUploadURL(repoKey, sha); - if (upload.kind !== 'ok') { - info(upload.kind === 'locked' - ? 'Another job is already uploading this snapshot; skipping' - : upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading'); - return; - } - const init = { - method: 'PUT', - headers: { - 'content-length': String(size), - 'content-type': 'application/x-tar' - }, - body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)), - duplex: 'half', - signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) - }; - const res = await fetch(upload.url, init); - if (!res.ok) { - throw new Error(`snapshot upload answered ${res.status}`); - } - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`); - info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`); - } - finally { - await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); - } + const manifest = { + repo_key: process.env['GITHUB_REPOSITORY_ID'], + sha, + git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git') + }; + const dir = manifestDir(); + await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true }); + await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest)); + info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`); } ;// CONCATENATED MODULE: ./src/git-source-provider.ts diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index 5f6ca85..0f8c379 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -1,10 +1,9 @@ /* eslint-disable i18n-text/no-en -- upstream convention */ import * as core from '@actions/core' -// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner -// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the -// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is -// uploading this snapshot (skip upload); else = fall back. +// Client for backend-core's git-mirror download-url endpoint, authed by the runner +// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable +// org; else = fall back. The upload half runs in the warpbuild-agent after the job. const API_TIMEOUT_MS = 10_000 @@ -20,12 +19,6 @@ export type SnapshotLookup = | {kind: 'disabled'} | {kind: 'error'} -export type UploadURLResult = - | {kind: 'ok'; url: string} - | {kind: 'disabled'} - | {kind: 'locked'} - | {kind: 'error'} - function baseUrl(): string { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') } @@ -65,40 +58,3 @@ export async function lookupSnapshot( return {kind: 'error'} } } - -export async function requestUploadURL( - repoKey: string, - sha: string -): Promise { - try { - const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, { - method: 'POST', - headers: { - authorization: authHeader(), - 'content-type': 'application/json' - }, - body: JSON.stringify({repo_key: repoKey, sha}), - signal: AbortSignal.timeout(API_TIMEOUT_MS) - }) - if (res.status === 200) { - const body = (await res.json()) as {url?: string} - if (body.url) { - return {kind: 'ok', url: body.url} - } - return {kind: 'error'} - } - if (res.status === 403) { - core.debug(`[wb-cache] upload-url answered 403 (disabled)`) - return {kind: 'disabled'} - } - if (res.status === 409) { - core.debug(`[wb-cache] upload-url answered 409 (locked)`) - return {kind: 'locked'} - } - core.debug(`[wb-cache] upload-url answered ${res.status}`) - return {kind: 'error'} - } catch (error) { - core.debug(`[wb-cache] upload-url failed: ${error}`) - return {kind: 'error'} - } -} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 879a43f..6b54721 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -15,10 +15,6 @@ import * as api from './backend-api.js' // immutable (no expiry). Fail-open: any error degrades to stock behavior. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 -const UPLOAD_TIMEOUT_MS = 15 * 60_000 - -// "hit " | "uploaded ", for e2e assertions. -export const CACHE_STATE_FILE = 'wb-cache-state' // Logged at debug (normal state outside WarpBuild); other reasons log at info. export const SKIP_NOT_WARPBUILD = @@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null { } // Runs right after `git init`; true = restored (caller skips the fetch). Never throws. +// Sets the `cache-hit` action output (true only on a restore). export async function setup(settings: IGitSourceSettings): Promise { + const restored = await setupImpl(settings) + core.setOutput('cache-hit', restored ? 'true' : 'false') + return restored +} + +async function setupImpl(settings: IGitSourceSettings): Promise { decision = 'off' const skipReason = getMirrorCacheSkipReason(settings) if (skipReason) { - if (skipReason === SKIP_NOT_WARPBUILD) { - core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`) - } else { - core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) - } + core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) return false } core.startGroup('WarpBuild: checkout snapshot cache') @@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise { return true } -// Runs after checkout; uploads the fetch result on a miss. Failures only warn. +// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the +// snapshot after the job exits (keeping tar+upload off the customer's billed time). export async function contribute(settings: IGitSourceSettings): Promise { if (decision !== 'miss') { return } - core.startGroup('WarpBuild: uploading checkout snapshot') try { - await uploadSnapshot(settings) + await writeSnapshotManifest(settings) } catch (error) { - core.warning(`Snapshot upload skipped: ${error}`) - } finally { - core.endGroup() + core.warning(`Snapshot manifest not written: ${error}`) } } @@ -242,80 +239,56 @@ async function restoreSnapshot( ]) } - await fs.promises.writeFile( - path.join(gitDir, CACHE_STATE_FILE), - `hit ${sha}\n` - ) return true } catch (error) { core.warning(`Snapshot restore failed: ${error}`) - // Reset .git to its freshly-init state. - await fs.promises.rm(path.join(gitDir, 'objects'), { - recursive: true, - force: true - }) - await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) - await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { - recursive: true - }) - await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { - recursive: true - }) + await resetGitObjects(gitDir) return false } finally { await fs.promises.rm(tmpTar, {force: true}) } } -async function uploadSnapshot(settings: IGitSourceSettings): Promise { - const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string - const sha = settings.commit - const gitDir = path.join(settings.repositoryPath, '.git') - const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`) - try { - const members = ['objects'] - if (fs.existsSync(path.join(gitDir, 'shallow'))) { - members.push('shallow') - } - await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]) - const size = (await fs.promises.stat(tmpTar)).size - - const upload = await api.requestUploadURL(repoKey, sha) - if (upload.kind !== 'ok') { - core.info( - upload.kind === 'locked' - ? 'Another job is already uploading this snapshot; skipping' - : upload.kind === 'disabled' - ? 'Snapshot cache is disabled by the backend; not uploading' - : 'Snapshot cache backend unavailable; not uploading' - ) - return - } - - const init: RequestInit & {duplex: 'half'} = { - method: 'PUT', - headers: { - 'content-length': String(size), - 'content-type': 'application/x-tar' - }, - body: Readable.toWeb( - fs.createReadStream(tmpTar) - ) as unknown as ReadableStream, - duplex: 'half', - signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) - } - const res = await fetch(upload.url, init) - if (!res.ok) { - throw new Error(`snapshot upload answered ${res.status}`) - } - await fs.promises.writeFile( - path.join(gitDir, CACHE_STATE_FILE), - `uploaded ${sha}\n` - ) - core.info( - `Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch` - ) - } finally { - await fs.promises.rm(tmpTar, {force: true}) - } +// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` +// creates, so a failed restore leaves no trace for the fallback fetch. +export async function resetGitObjects(gitDir: string): Promise { + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }) + await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) +} + +// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which +// is /_work/_temp — host-visible to the agent (it knows the runner dir). +export function manifestDir(): string { + const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir() + return path.join(runnerTemp, 'wb-snapshots') +} + +// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. +async function writeSnapshotManifest( + settings: IGitSourceSettings +): Promise { + const sha = settings.commit + const manifest = { + repo_key: process.env['GITHUB_REPOSITORY_ID'] as string, + sha, + git_dir: path.join(settings.repositoryPath, '.git') + } + const dir = manifestDir() + await fs.promises.mkdir(dir, {recursive: true}) + await fs.promises.writeFile( + path.join(dir, `${sha}.json`), + JSON.stringify(manifest) + ) + core.info( + `Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job` + ) } From 0c607adfe50b391d030f6dd15cf791e5026ceb12 Mon Sep 17 00:00:00 2001 From: darshanime Date: Mon, 13 Jul 2026 08:48:11 +0530 Subject: [PATCH 06/13] get deltas from github --- __test__/warpbuild-mirror.test.ts | 270 +++++++----------- dist/index.js | 436 ++++++++++++++++++------------ src/git-source-provider.ts | 19 +- src/warpbuild/backend-api.ts | 98 +++++-- src/warpbuild/mirror-cache.ts | 423 ++++++++++++++++------------- 5 files changed, 706 insertions(+), 540 deletions(-) diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 7d280fd..e6250e9 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,26 +6,29 @@ import { afterEach, afterAll } from '@jest/globals' -import * as exec from '@actions/exec' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' import { SKIP_NOT_WARPBUILD, - assertSafeTarMembers, - computeDestinationRef, + computeRefKey, getMirrorCacheSkipReason, - manifestDir, resetGitObjects } from '../src/warpbuild/mirror-cache.js' -import {lookupSnapshot} from '../src/warpbuild/backend-api.js' +import { + lookupRestore, + requestBaseUpload, + requestBranchUpload +} from '../src/warpbuild/backend-api.js' import {IGitSourceSettings} from '../src/git-source-settings.js' const WB_ENV = [ 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', 'WARPBUILD_HOST_URL', 'GITHUB_REPOSITORY_ID', - 'GITHUB_REPOSITORY' + 'GITHUB_REPOSITORY', + 'GITHUB_REF', + 'GITHUB_BASE_REF' ] const savedEnv: {[key: string]: string | undefined} = {} for (const key of WB_ENV) { @@ -57,9 +60,11 @@ function setWarpBuildEnv(): void { process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' process.env['GITHUB_REPOSITORY_ID'] = '123456789' process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' + delete process.env['GITHUB_REF'] + delete process.env['GITHUB_BASE_REF'] } -describe('warpbuild snapshot cache', () => { +describe('warpbuild mirror cache', () => { beforeEach(() => { setWarpBuildEnv() }) @@ -75,9 +80,7 @@ describe('warpbuild snapshot cache', () => { }) describe('getMirrorCacheSkipReason', () => { - const winOnly = process.platform === 'win32' ? it.skip : it - - winOnly('returns null for the default checkout shape', () => { + it('returns null for the default checkout shape', () => { expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() }) @@ -86,7 +89,7 @@ describe('warpbuild snapshot cache', () => { expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) }) - it('reports repository: inputs that are not the workflow repo', () => { + it('reports repository inputs that are not the workflow repo', () => { expect( getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) ).toBe( @@ -103,19 +106,15 @@ describe('warpbuild snapshot cache', () => { ) }) - it('only serves fetch-depth 1', () => { - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( - 'fetch-depth is 0, cache only serves fetch-depth 1' - ) - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( - 'fetch-depth is 50, cache only serves fetch-depth 1' - ) + it('serves any fetch-depth and fetch-tags (mirror is full history)', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull() + expect( + getMirrorCacheSkipReason(settingsFor({fetchTags: true})) + ).toBeNull() }) - it('skips fetch-tags, filters, sparse and lfs checkouts', () => { - expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe( - 'fetch-tags is enabled' - ) + it('skips filters, sparse and lfs checkouts', () => { expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( 'a fetch filter is configured' ) @@ -123,18 +122,36 @@ describe('warpbuild snapshot cache', () => { getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) ).toBe('sparse checkout is configured') expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( - 'lfs is enabled (lfs objects are not in the snapshot)' + 'lfs is enabled (lfs objects are not in the mirror)' ) }) + }) - winOnly('accepts sha-only checkouts (empty ref)', () => { - expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() + describe('computeRefKey', () => { + it('keys on the pushed branch (GITHUB_REF)', () => { + process.env['GITHUB_REF'] = 'refs/heads/feature/x' + expect(computeRefKey(settingsFor())).toBe('feature/x') }) - it('skips unqualified refs', () => { - expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( - "ref 'main' has no cacheable destination ref" - ) + it('keys on the base branch for pull requests (merge sha is synthetic)', () => { + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + process.env['GITHUB_BASE_REF'] = 'main' + expect(computeRefKey(settingsFor())).toBe('main') + }) + + it('is empty for tags and bare pull refs (base only, no roll)', () => { + process.env['GITHUB_REF'] = 'refs/tags/v1.2.3' + expect(computeRefKey(settingsFor())).toBe('') + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + expect(computeRefKey(settingsFor())).toBe('') + }) + + it('is empty for a detached sha checkout', () => { + expect(computeRefKey(settingsFor({ref: SHA}))).toBe('') + }) + + it('passes through an already-short branch name', () => { + expect(computeRefKey(settingsFor({ref: 'main'}))).toBe('main') }) }) @@ -144,7 +161,6 @@ describe('warpbuild snapshot cache', () => { path.join(os.tmpdir(), 'wb-reset-') ) try { - // Simulate a partially-restored .git: stray objects, a pack, a shallow file. await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { recursive: true }) @@ -152,179 +168,95 @@ describe('warpbuild snapshot cache', () => { path.join(gitDir, 'objects', 'ab'), 'junk loose object' ) - await fs.promises.writeFile( - path.join(gitDir, 'objects', 'pack', 'pack-x.pack'), - 'junk pack' - ) await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n') await resetGitObjects(gitDir) - // shallow gone; objects/ holds only the two empty init dirs. expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false) expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false) expect( (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() ).toEqual(['info', 'pack']) - expect( - await fs.promises.readdir(path.join(gitDir, 'objects', 'pack')) - ).toEqual([]) - expect( - await fs.promises.readdir(path.join(gitDir, 'objects', 'info')) - ).toEqual([]) } finally { await fs.promises.rm(gitDir, {recursive: true, force: true}) } }) - - it('is a no-op-safe on an already-clean .git', async () => { - const gitDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'wb-reset-') - ) - try { - await expect(resetGitObjects(gitDir)).resolves.toBeUndefined() - expect( - (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() - ).toEqual(['info', 'pack']) - } finally { - await fs.promises.rm(gitDir, {recursive: true, force: true}) - } - }) - }) - - describe('assertSafeTarMembers', () => { - async function makeTar( - dir: string, - members: string[], - writeFiles = true - ): Promise { - if (writeFiles) { - for (const m of members) { - const abs = path.join(dir, m) - await fs.promises.mkdir(path.dirname(abs), {recursive: true}) - await fs.promises.writeFile(abs, 'x') - } - } - const tar = path.join(dir, 'out.tar') - await exec.exec('tar', ['-cf', tar, '-C', dir, ...members]) - return tar - } - - it('accepts objects/ and shallow members', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow']) - await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined() - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - - it('rejects members outside objects/ and shallow (e.g. hooks)', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit']) - await expect(assertSafeTarMembers(tar)).rejects.toThrow( - /unexpected snapshot tar member/ - ) - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - - it('rejects path traversal members', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - await fs.promises.mkdir(path.join(dir, 'sub')) - const tar = path.join(dir, 'evil.tar') - // -P preserves the ../ member instead of stripping it. - await fs.promises.writeFile(path.join(dir, 'evil'), 'x') - await exec.exec('tar', [ - '-cPf', - tar, - '-C', - path.join(dir, 'sub'), - '../evil' - ]) - await expect(assertSafeTarMembers(tar)).rejects.toThrow( - /unexpected snapshot tar member/ - ) - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - }) - - describe('computeDestinationRef', () => { - it('maps the refs the fetch would have created', () => { - expect(computeDestinationRef('refs/heads/main')).toBe( - 'refs/remotes/origin/main' - ) - expect(computeDestinationRef('refs/pull/42/merge')).toBe( - 'refs/remotes/pull/42/merge' - ) - expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3') - expect(computeDestinationRef('')).toBe('') - expect(computeDestinationRef('main')).toBeNull() - }) }) describe('backend api http contract', () => { const realFetch = globalThis.fetch let lastUrl = '' + let lastInit: RequestInit | undefined afterEach(() => { globalThis.fetch = realFetch }) function stubFetch(status: number, body: unknown): void { - globalThis.fetch = (async (input: unknown) => { + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { lastUrl = String(input) + lastInit = init return new Response(JSON.stringify(body), {status}) }) as typeof fetch } - it('maps 200 to hit and keys by sha', async () => { - stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) - const result = await lookupSnapshot('123', SHA) - expect(result.kind).toBe('hit') - expect(lastUrl).toContain(`sha=${SHA}`) + it('lookupRestore maps 200 to a restore plan keyed by ref', async () => { + stubFetch(200, { + base: {url: 'https://s3/base', size_bytes: 42}, + branch: {url: 'https://s3/branch'} + }) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.base.url).toBe('https://s3/base') + expect(result.branch?.url).toBe('https://s3/branch') + } + expect(lastUrl).toContain('ref=main') }) - it('maps 404 to miss (agent uploads after the job)', async () => { - stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') - }) - - it('maps 403 to disabled (backend kill switch)', async () => { - stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') - }) - - it('maps other statuses and network failures to error', async () => { - stubFetch(500, {}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('error') - globalThis.fetch = (async () => { - throw new Error('boom') - }) as typeof fetch - expect((await lookupSnapshot('123', SHA)).kind).toBe('error') - }) - }) - - describe('manifestDir', () => { - const saved = process.env['RUNNER_TEMP'] - afterEach(() => { - if (saved === undefined) { - delete process.env['RUNNER_TEMP'] - } else { - process.env['RUNNER_TEMP'] = saved + it('lookupRestore maps a null branch (base but no delta yet)', async () => { + stubFetch(200, {base: {url: 'https://s3/base'}, branch: null}) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.branch).toBeNull() } }) - it('places manifests under RUNNER_TEMP/wb-snapshots', () => { - process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x') - expect(manifestDir()).toBe( - path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots') + it('lookupRestore maps 404 to cold, 403 to disabled, 500 to error', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupRestore('123', 'main')).kind).toBe('cold') + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupRestore('123', 'main')).kind).toBe('disabled') + stubFetch(500, {}) + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('lookupRestore maps network failure to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('requestBaseUpload maps 200 to a grant, 409 to locked', async () => { + stubFetch(200, {url: 'https://s3/put-base'}) + const grant = await requestBaseUpload('123') + expect(grant.kind).toBe('grant') + if (grant.kind === 'grant') { + expect(grant.url).toBe('https://s3/put-base') + } + expect(lastInit?.method).toBe('POST') + stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) + expect((await requestBaseUpload('123')).kind).toBe('locked') + }) + + it('requestBranchUpload maps 200 to a grant, 403 to disabled', async () => { + stubFetch(200, {url: 'https://s3/put-branch'}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe('grant') + stubFetch(403, {}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe( + 'disabled' ) }) }) diff --git a/dist/index.js b/dist/index.js index 0a42bda..3bbc325 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41697,9 +41697,11 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts /* eslint-disable i18n-text/no-en -- upstream convention */ -// Client for backend-core's git-mirror download-url endpoint, authed by the runner -// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable -// org; else = fall back. The upload half runs in the warpbuild-agent after the job. +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41707,30 +41709,71 @@ function backend_api_baseUrl() { function authHeader() { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; } -async function lookupSnapshot(repoKey, sha) { +function backend_api_endpoint(p) { + return `${backend_api_baseUrl()}/api/v1/git-mirrors/${p}`; +} +async function lookupRestore(repoKey, ref) { try { - const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { + const res = await fetch(`${backend_api_endpoint('restore-url')}?repo_key=${encodeURIComponent(repoKey)}&ref=${encodeURIComponent(ref)}`, { headers: { authorization: authHeader() }, signal: AbortSignal.timeout(API_TIMEOUT_MS) }); if (res.status === 200) { - return { kind: 'hit', info: (await res.json()) }; + const body = (await res.json()); + return { kind: 'restore', base: body.base, branch: body.branch ?? null }; } if (res.status === 404) { - return { kind: 'miss' }; + return { kind: 'cold' }; } if (res.status === 403) { - core_debug(`[wb-cache] download-url answered 403 (disabled)`); + core_debug('[wb-cache] restore-url answered 403 (disabled)'); return { kind: 'disabled' }; } - core_debug(`[wb-cache] download-url answered ${res.status}`); + core_debug(`[wb-cache] restore-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-cache] download-url failed: ${error}`); + core_debug(`[wb-cache] restore-url failed: ${error}`); return { kind: 'error' }; } } +async function requestUpload(p, body) { + try { + const res = await fetch(backend_api_endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const b = (await res.json()); + return { kind: 'grant', url: b.url }; + } + if (res.status === 409) { + return { kind: 'locked' }; + } + if (res.status === 403) { + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] ${p} answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] ${p} failed: ${error}`); + return { kind: 'error' }; + } +} +// Cold repo: request the single-flight grant to build + upload the base mirror. +async function requestBaseUpload(repoKey) { + return requestUpload('base/upload-url', { repo_key: repoKey }); +} +// Warm repo: request the grant to overwrite this branch's delta bundle. +async function requestBranchUpload(repoKey, ref, sha) { + return requestUpload('branch/upload-url', { repo_key: repoKey, ref, sha }); +} ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ @@ -41743,23 +41786,29 @@ async function lookupSnapshot(repoKey, sha) { -// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch -// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are -// immutable (no expiry). Fail-open: any error degrades to stock behavior. +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; -// Logged at debug (normal state outside WarpBuild); other reasons log at info. +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Logged at info when off; the WARPBUILD_* env is only present on our runners. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; -let decision = 'off'; -// Null = attempt the cache; else a reason to log. Only the default checkout shape -// is served. +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base'; +const BRANCH_REFNS = 'refs/wb/branch'; +const UPLOAD_TIP_REF = 'refs/wb/tip'; +let plan = { mode: 'off', repoKey: '', refKey: '' }; +// Null = attempt the cache; else a reason to log. The mirror serves full history for any +// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. function getMirrorCacheSkipReason(settings) { if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { return SKIP_NOT_WARPBUILD; } - const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; - if (!repoKey) { + if (!process.env['GITHUB_REPOSITORY_ID']) { return 'GITHUB_REPOSITORY_ID is not set'; } const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; @@ -41773,12 +41822,6 @@ function getMirrorCacheSkipReason(settings) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on'; } - if (settings.fetchDepth !== 1) { - return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`; - } - if (settings.fetchTags) { - return 'fetch-tags is enabled'; - } if (settings.filter) { return 'a fetch filter is configured'; } @@ -41786,51 +41829,49 @@ function getMirrorCacheSkipReason(settings) { return 'sparse checkout is configured'; } if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the snapshot)'; - } - if (settings.ref && computeDestinationRef(settings.ref) === null) { - return `ref '${settings.ref}' has no cacheable destination ref`; + return 'lfs is enabled (lfs objects are not in the mirror)'; } return null; } -// The local ref the fetch would have created ('' = none needed, null = uncacheable). -function computeDestinationRef(ref) { - if (!ref) { +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +function computeRefKey(settings) { + const baseRef = process.env['GITHUB_BASE_REF']; // set on pull_request events + if (baseRef) { + return baseRef; + } + const ref = process.env['GITHUB_REF'] || settings.ref || ''; + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length); + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { return ''; } - const upper = ref.toUpperCase(); - if (upper.startsWith('REFS/HEADS/')) { - return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`; - } - if (upper.startsWith('REFS/PULL/')) { - return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`; - } - if (upper.startsWith('REFS/TAGS/')) { - return ref; - } - return null; + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref; } -// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. -// Sets the `cache-hit` action output (true only on a restore). +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). async function setup(settings) { - const restored = await setupImpl(settings); - setOutput('cache-hit', restored ? 'true' : 'false'); - return restored; + const mode = await setupImpl(settings); + setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false'); + return mode; } async function setupImpl(settings) { - decision = 'off'; + plan = { mode: 'off', repoKey: '', refKey: '' }; const skipReason = getMirrorCacheSkipReason(settings); if (skipReason) { - info(`WarpBuild snapshot cache skipped: ${skipReason}`); - return false; + info(`WarpBuild mirror cache skipped: ${skipReason}`); + return 'off'; } - startGroup('WarpBuild: checkout snapshot cache'); + startGroup('WarpBuild: checkout mirror cache'); try { return await setupInner(settings); } catch (error) { - warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); - return false; + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + return 'off'; } finally { endGroup(); @@ -41838,140 +41879,196 @@ async function setupImpl(settings) { } async function setupInner(settings) { const repoKey = process.env['GITHUB_REPOSITORY_ID']; - const sha = settings.commit; - // The restore/upload paths shell out to `tar`; without it, fall back cleanly. - if (!(await which('tar', false))) { - info('tar not found on PATH; using standard checkout'); - return false; - } - const lookup = await lookupSnapshot(repoKey, sha); + const refKey = computeRefKey(settings); + plan = { mode: 'off', repoKey, refKey }; + const lookup = await lookupRestore(repoKey, refKey); if (lookup.kind === 'disabled') { - info('Snapshot cache is disabled by the backend for this organization'); - return false; + info('Mirror cache is disabled by the backend for this organization'); + return 'off'; } if (lookup.kind === 'error') { - info('Snapshot cache backend unavailable; using standard checkout'); - return false; + info('Mirror cache backend unavailable; using standard checkout'); + return 'off'; } - if (lookup.kind === 'miss') { - info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); - decision = 'miss'; - return false; + if (lookup.kind === 'cold') { + const grant = await requestBaseUpload(repoKey); + if (grant.kind === 'grant') { + info('Cold repo: building the base mirror this run (full fetch, then upload)'); + plan = { mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url }; + return 'cold-build'; + } + info('Cold repo: base build already in progress elsewhere; using standard checkout'); + return 'off'; } - info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); - if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { - return false; + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS); + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS); + } + catch (error) { + info(`Branch delta seed skipped (${error}); base only`); + } } - info('Snapshot restored; skipping the GitHub fetch entirely'); - return true; + info(`Seeded base mirror${lookup.branch ? ' + branch delta' : ''}; the GitHub fetch will be a tip delta`); + plan = { mode: 'seeded', repoKey, refKey }; + return 'seeded'; } -// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the -// snapshot after the job exits (keeping tar+upload off the customer's billed time). +// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) +// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. async function contribute(settings) { - if (decision !== 'miss') { - return; - } try { - await writeSnapshotManifest(settings); + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings); + } + else if (plan.mode === 'seeded') { + await uploadBranchDelta(settings); + } } catch (error) { - warning(`Snapshot manifest not written: ${error}`); + warning(`WarpBuild mirror upload skipped: ${error}`); } } -// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` -// component — the tar is remote and extracted into .git, so a crafted member could -// escape (e.g. hooks/, ../) and run code during checkout. -async function assertSafeTarMembers(tar) { - let listing = ''; - await exec_exec('tar', ['-tf', tar], { - silent: true, - listeners: { stdout: (d) => (listing += d.toString()) } - }); - for (const raw of listing.split('\n')) { - const member = raw.trim(); - if (!member) { - continue; - } - const top = member.replace(/^\.\//, '').split('/')[0]; - if (member.startsWith('/') || - member.split('/').includes('..') || - (top !== 'objects' && top !== 'shallow')) { - throw new Error(`unexpected snapshot tar member: ${member}`); - } - } -} -async function restoreSnapshot(settings, url, sha) { - const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle(repoPath, url, refNs) { + const tmp = tempBundlePath('seed'); try { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }); - if (!res.ok || !res.body) { - throw new Error(`snapshot download answered ${res.status}`); - } - await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); - await assertSafeTarMembers(tmpTar); - await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); - const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); - if (check !== 0) { - throw new Error(`restored snapshot does not contain ${sha}`); - } - // The ref the skipped fetch would have created; upstream's verification needs it. - const dstRef = computeDestinationRef(settings.ref); - if (dstRef) { - await exec_exec('git', [ - '-C', - settings.repositoryPath, - 'update-ref', - dstRef, - sha - ]); - } - return true; - } - catch (error) { - warning(`Snapshot restore failed: ${error}`); - await resetGitObjects(gitDir); - return false; + await downloadTo(url, tmp); + await exec_exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]); } finally { - await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); } } -// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` -// creates, so a failed restore leaves no trace for the fallback fetch. +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings) { + if (!plan.baseUploadUrl) { + return; + } + const tmp = tempBundlePath('base'); + try { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]); + await httpPut(plan.baseUploadUrl, tmp); + info('Uploaded base mirror'); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } +} +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings) { + if (!plan.refKey) { + return; // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath; + const sha = settings.commit; + const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha); + if (grant.kind !== 'grant') { + info(`Branch delta upload skipped (${grant.kind})`); + return; + } + const excludes = await baseRefExcludes(repoPath); + if (excludes.length === 0) { + info('No base refs to diff against; branch delta skipped'); + return; + } + const tmp = tempBundlePath('branch'); + try { + await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); + await exec_exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + ...excludes + ]); + await httpPut(grant.url, tmp); + info(`Uploaded branch delta for '${plan.refKey}'`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], { + ignoreReturnCode: true + }); + } +} +// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle +// base-relative (and therefore order-free). +async function baseRefExcludes(repoPath) { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out + .split('\n') + .map(s => s.trim()) + .filter(Boolean); +} +async function downloadTo(url, dest) { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(dest)); +} +async function httpPut(url, file) { + const stat = await external_fs_namespaceObject.promises.stat(file); + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: external_fs_namespaceObject.createReadStream(file), + duplex: 'half', + headers: { 'content-length': String(stat.size) }, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + }); + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`); + } +} +function tempBundlePath(tag) { + return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`); +} +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. async function resetGitObjects(gitDir) { - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + await fs.promises.rm(path.join(gitDir, 'objects'), { recursive: true, force: true }); - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + await fs.promises.rm(path.join(gitDir, 'shallow'), { force: true }); + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { recursive: true }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { recursive: true }); } -// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which -// is /_work/_temp — host-visible to the agent (it knows the runner dir). -function manifestDir() { - const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir(); - return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots'); -} -// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. -async function writeSnapshotManifest(settings) { - const sha = settings.commit; - const manifest = { - repo_key: process.env['GITHUB_REPOSITORY_ID'], - sha, - git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git') - }; - const dir = manifestDir(); - await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true }); - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest)); - info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`); +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +async function hasGit() { + return Boolean(await io.which('git', false)); } ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -42007,7 +42104,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; - let warpbuildRestored = false; + let warpbuildMode = 'off'; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -42059,7 +42156,7 @@ async function getSource(settings) { await git.remoteAdd('origin', repositoryUrl); endGroup(); // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped - warpbuildRestored = await setup(settings); + warpbuildMode = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -42099,10 +42196,17 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (warpbuildRestored) { - info('Skipping fetch: checkout was restored from the snapshot cache'); + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. + const refSpec = getRefSpec(settings.ref, settings.commit); + await git.fetch(refSpec, fetchOptions); + if (!(await testRef(git, settings.ref, settings.commit))) { + throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.`); + } } - else if (settings.fetchDepth <= 0) { + else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 3b5adbc..fbdf9d7 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -41,7 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null - let warpbuildRestored = false + let warpbuildMode: warpbuildMirror.MirrorMode = 'off' try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -134,7 +134,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped - warpbuildRestored = await warpbuildMirror.setup(settings) + warpbuildMode = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -190,9 +190,18 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (warpbuildRestored) { - core.info('Skipping fetch: checkout was restored from the snapshot cache') - } else if (settings.fetchDepth <= 0) { + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + throw new Error( + `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.` + ) + } + } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index 0f8c379..b052abc 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -1,21 +1,30 @@ /* eslint-disable i18n-text/no-en -- upstream convention */ import * as core from '@actions/core' -// Client for backend-core's git-mirror download-url endpoint, authed by the runner -// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable -// org; else = fall back. The upload half runs in the warpbuild-agent after the job. +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. const API_TIMEOUT_MS = 10_000 -export interface SnapshotDownloadInfo { +export interface Presigned { url: string - size_bytes: number - created_at: string + size_bytes?: number } -export type SnapshotLookup = - | {kind: 'hit'; info: SnapshotDownloadInfo} - | {kind: 'miss'} +// GET restore-url: what to seed before the delta fetch. 'cold' = no base yet. +export type RestoreLookup = + | {kind: 'restore'; base: Presigned; branch: Presigned | null} + | {kind: 'cold'} + | {kind: 'disabled'} + | {kind: 'error'} + +// POST *upload-url: a presigned PUT, or a signal to skip (locked/disabled). +export type UploadGrant = + | {kind: 'grant'; url: string} + | {kind: 'locked'} | {kind: 'disabled'} | {kind: 'error'} @@ -27,34 +36,85 @@ function authHeader(): string { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` } -export async function lookupSnapshot( +function endpoint(p: string): string { + return `${baseUrl()}/api/v1/git-mirrors/${p}` +} + +export async function lookupRestore( repoKey: string, - sha: string -): Promise { + ref: string +): Promise { try { const res = await fetch( - `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( + `${endpoint('restore-url')}?repo_key=${encodeURIComponent( repoKey - )}&sha=${encodeURIComponent(sha)}`, + )}&ref=${encodeURIComponent(ref)}`, { headers: {authorization: authHeader()}, signal: AbortSignal.timeout(API_TIMEOUT_MS) } ) if (res.status === 200) { - return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} + const body = (await res.json()) as { + base: Presigned + branch: Presigned | null + } + return {kind: 'restore', base: body.base, branch: body.branch ?? null} } if (res.status === 404) { - return {kind: 'miss'} + return {kind: 'cold'} } if (res.status === 403) { - core.debug(`[wb-cache] download-url answered 403 (disabled)`) + core.debug('[wb-cache] restore-url answered 403 (disabled)') return {kind: 'disabled'} } - core.debug(`[wb-cache] download-url answered ${res.status}`) + core.debug(`[wb-cache] restore-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-cache] download-url failed: ${error}`) + core.debug(`[wb-cache] restore-url failed: ${error}`) return {kind: 'error'} } } + +async function requestUpload(p: string, body: object): Promise { + try { + const res = await fetch(endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const b = (await res.json()) as {url: string} + return {kind: 'grant', url: b.url} + } + if (res.status === 409) { + return {kind: 'locked'} + } + if (res.status === 403) { + return {kind: 'disabled'} + } + core.debug(`[wb-cache] ${p} answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] ${p} failed: ${error}`) + return {kind: 'error'} + } +} + +// Cold repo: request the single-flight grant to build + upload the base mirror. +export async function requestBaseUpload(repoKey: string): Promise { + return requestUpload('base/upload-url', {repo_key: repoKey}) +} + +// Warm repo: request the grant to overwrite this branch's delta bundle. +export async function requestBranchUpload( + repoKey: string, + ref: string, + sha: string +): Promise { + return requestUpload('branch/upload-url', {repo_key: repoKey, ref, sha}) +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 6b54721..14ddefe 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -10,22 +10,38 @@ import {pipeline} from 'stream/promises' import {IGitSourceSettings} from '../git-source-settings.js' import * as api from './backend-api.js' -// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch -// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are -// immutable (no expiry). Fail-open: any error degrades to stock behavior. +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 -// Logged at debug (normal state outside WarpBuild); other reasons log at info. +// Logged at info when off; the WARPBUILD_* env is only present on our runners. export const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)' const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ -let decision: 'off' | 'miss' = 'off' +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base' +const BRANCH_REFNS = 'refs/wb/branch' +const UPLOAD_TIP_REF = 'refs/wb/tip' -// Null = attempt the cache; else a reason to log. Only the default checkout shape -// is served. +export type MirrorMode = 'off' | 'seeded' | 'cold-build' + +interface Plan { + mode: MirrorMode + repoKey: string + refKey: string + baseUploadUrl?: string // cold-build only +} +let plan: Plan = {mode: 'off', repoKey: '', refKey: ''} + +// Null = attempt the cache; else a reason to log. The mirror serves full history for any +// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { @@ -35,8 +51,7 @@ export function getMirrorCacheSkipReason( ) { return SKIP_NOT_WARPBUILD } - const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' - if (!repoKey) { + if (!process.env['GITHUB_REPOSITORY_ID']) { return 'GITHUB_REPOSITORY_ID is not set' } const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` @@ -53,12 +68,6 @@ export function getMirrorCacheSkipReason( if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on' } - if (settings.fetchDepth !== 1) { - return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1` - } - if (settings.fetchTags) { - return 'fetch-tags is enabled' - } if (settings.filter) { return 'a fetch filter is configured' } @@ -66,191 +75,266 @@ export function getMirrorCacheSkipReason( return 'sparse checkout is configured' } if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the snapshot)' - } - if (settings.ref && computeDestinationRef(settings.ref) === null) { - return `ref '${settings.ref}' has no cacheable destination ref` + return 'lfs is enabled (lfs objects are not in the mirror)' } return null } -// The local ref the fetch would have created ('' = none needed, null = uncacheable). -export function computeDestinationRef(ref: string): string | null { - if (!ref) { +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +export function computeRefKey(settings: IGitSourceSettings): string { + const baseRef = process.env['GITHUB_BASE_REF'] // set on pull_request events + if (baseRef) { + return baseRef + } + const ref = process.env['GITHUB_REF'] || settings.ref || '' + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length) + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { return '' } - const upper = ref.toUpperCase() - if (upper.startsWith('REFS/HEADS/')) { - return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}` - } - if (upper.startsWith('REFS/PULL/')) { - return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}` - } - if (upper.startsWith('REFS/TAGS/')) { - return ref - } - return null + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref } -// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. -// Sets the `cache-hit` action output (true only on a restore). -export async function setup(settings: IGitSourceSettings): Promise { - const restored = await setupImpl(settings) - core.setOutput('cache-hit', restored ? 'true' : 'false') - return restored +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). +export async function setup(settings: IGitSourceSettings): Promise { + const mode = await setupImpl(settings) + core.setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false') + return mode } -async function setupImpl(settings: IGitSourceSettings): Promise { - decision = 'off' +async function setupImpl(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} const skipReason = getMirrorCacheSkipReason(settings) if (skipReason) { - core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) - return false + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + return 'off' } - core.startGroup('WarpBuild: checkout snapshot cache') + core.startGroup('WarpBuild: checkout mirror cache') try { return await setupInner(settings) } catch (error) { core.warning( - `WarpBuild snapshot cache unavailable, using standard checkout: ${error}` + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` ) - return false + return 'off' } finally { core.endGroup() } } -async function setupInner(settings: IGitSourceSettings): Promise { +async function setupInner(settings: IGitSourceSettings): Promise { const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string - const sha = settings.commit - - // The restore/upload paths shell out to `tar`; without it, fall back cleanly. - if (!(await io.which('tar', false))) { - core.info('tar not found on PATH; using standard checkout') - return false - } - - const lookup = await api.lookupSnapshot(repoKey, sha) + const refKey = computeRefKey(settings) + plan = {mode: 'off', repoKey, refKey} + const lookup = await api.lookupRestore(repoKey, refKey) if (lookup.kind === 'disabled') { - core.info('Snapshot cache is disabled by the backend for this organization') - return false + core.info('Mirror cache is disabled by the backend for this organization') + return 'off' } if (lookup.kind === 'error') { - core.info('Snapshot cache backend unavailable; using standard checkout') - return false + core.info('Mirror cache backend unavailable; using standard checkout') + return 'off' } - if (lookup.kind === 'miss') { + if (lookup.kind === 'cold') { + const grant = await api.requestBaseUpload(repoKey) + if (grant.kind === 'grant') { + core.info( + 'Cold repo: building the base mirror this run (full fetch, then upload)' + ) + plan = {mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url} + return 'cold-build' + } core.info( - `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` + 'Cold repo: base build already in progress elsewhere; using standard checkout' ) - decision = 'miss' - return false + return 'off' } - core.info( - `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` - ) - if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { - return false + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS) + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS) + } catch (error) { + core.info(`Branch delta seed skipped (${error}); base only`) + } } - core.info('Snapshot restored; skipping the GitHub fetch entirely') - return true + core.info( + `Seeded base mirror${ + lookup.branch ? ' + branch delta' : '' + }; the GitHub fetch will be a tip delta` + ) + plan = {mode: 'seeded', repoKey, refKey} + return 'seeded' } -// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the -// snapshot after the job exits (keeping tar+upload off the customer's billed time). +// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) +// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. export async function contribute(settings: IGitSourceSettings): Promise { - if (decision !== 'miss') { + try { + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings) + } else if (plan.mode === 'seeded') { + await uploadBranchDelta(settings) + } + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } +} + +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle( + repoPath: string, + url: string, + refNs: string +): Promise { + const tmp = tempBundlePath('seed') + try { + await downloadTo(url, tmp) + await exec.exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]) + } finally { + await fs.promises.rm(tmp, {force: true}) + } +} + +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings: IGitSourceSettings): Promise { + if (!plan.baseUploadUrl) { return } + const tmp = tempBundlePath('base') try { - await writeSnapshotManifest(settings) - } catch (error) { - core.warning(`Snapshot manifest not written: ${error}`) - } -} - -// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` -// component — the tar is remote and extracted into .git, so a crafted member could -// escape (e.g. hooks/, ../) and run code during checkout. -export async function assertSafeTarMembers(tar: string): Promise { - let listing = '' - await exec.exec('tar', ['-tf', tar], { - silent: true, - listeners: {stdout: (d: Buffer) => (listing += d.toString())} - }) - for (const raw of listing.split('\n')) { - const member = raw.trim() - if (!member) { - continue - } - const top = member.replace(/^\.\//, '').split('/')[0] - if ( - member.startsWith('/') || - member.split('/').includes('..') || - (top !== 'objects' && top !== 'shallow') - ) { - throw new Error(`unexpected snapshot tar member: ${member}`) - } - } -} - -async function restoreSnapshot( - settings: IGitSourceSettings, - url: string, - sha: string -): Promise { - const gitDir = path.join(settings.repositoryPath, '.git') - const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`) - try { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }) - if (!res.ok || !res.body) { - throw new Error(`snapshot download answered ${res.status}`) - } - await pipeline( - Readable.fromWeb(res.body as import('stream/web').ReadableStream), - fs.createWriteStream(tmpTar) - ) - await assertSafeTarMembers(tmpTar) - await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) - - const check = await exec.exec( - 'git', - ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], - {ignoreReturnCode: true} - ) - if (check !== 0) { - throw new Error(`restored snapshot does not contain ${sha}`) - } - - // The ref the skipped fetch would have created; upstream's verification needs it. - const dstRef = computeDestinationRef(settings.ref) - if (dstRef) { - await exec.exec('git', [ - '-C', - settings.repositoryPath, - 'update-ref', - dstRef, - sha - ]) - } - - return true - } catch (error) { - core.warning(`Snapshot restore failed: ${error}`) - await resetGitObjects(gitDir) - return false + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]) + await httpPut(plan.baseUploadUrl, tmp) + core.info('Uploaded base mirror') } finally { - await fs.promises.rm(tmpTar, {force: true}) + await fs.promises.rm(tmp, {force: true}) } } -// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` -// creates, so a failed restore leaves no trace for the fallback fetch. +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings: IGitSourceSettings): Promise { + if (!plan.refKey) { + return // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath + const sha = settings.commit + + const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha) + if (grant.kind !== 'grant') { + core.info(`Branch delta upload skipped (${grant.kind})`) + return + } + + const excludes = await baseRefExcludes(repoPath) + if (excludes.length === 0) { + core.info('No base refs to diff against; branch delta skipped') + return + } + + const tmp = tempBundlePath('branch') + try { + await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) + await exec.exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + ...excludes + ]) + await httpPut(grant.url, tmp) + core.info(`Uploaded branch delta for '${plan.refKey}'`) + } finally { + await fs.promises.rm(tmp, {force: true}) + await exec.exec( + 'git', + ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], + { + ignoreReturnCode: true + } + ) + } +} + +// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle +// base-relative (and therefore order-free). +async function baseRefExcludes(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + return out + .split('\n') + .map(s => s.trim()) + .filter(Boolean) +} + +async function downloadTo(url: string, dest: string): Promise { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(dest) + ) +} + +async function httpPut(url: string, file: string): Promise { + const stat = await fs.promises.stat(file) + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: fs.createReadStream(file) as any, + duplex: 'half', + headers: {'content-length': String(stat.size)}, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + } as any) + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`) + } +} + +function tempBundlePath(tag: string): string { + return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`) +} + +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. export async function resetGitObjects(gitDir: string): Promise { await fs.promises.rm(path.join(gitDir, 'objects'), { recursive: true, @@ -265,30 +349,7 @@ export async function resetGitObjects(gitDir: string): Promise { }) } -// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which -// is /_work/_temp — host-visible to the agent (it knows the runner dir). -export function manifestDir(): string { - const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir() - return path.join(runnerTemp, 'wb-snapshots') -} - -// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. -async function writeSnapshotManifest( - settings: IGitSourceSettings -): Promise { - const sha = settings.commit - const manifest = { - repo_key: process.env['GITHUB_REPOSITORY_ID'] as string, - sha, - git_dir: path.join(settings.repositoryPath, '.git') - } - const dir = manifestDir() - await fs.promises.mkdir(dir, {recursive: true}) - await fs.promises.writeFile( - path.join(dir, `${sha}.json`), - JSON.stringify(manifest) - ) - core.info( - `Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job` - ) +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +export async function hasGit(): Promise { + return Boolean(await io.which('git', false)) } From 3f843303fa116a041a9dd1eb01691bd36c529d00 Mon Sep 17 00:00:00 2001 From: darshanime Date: Mon, 13 Jul 2026 13:32:03 +0530 Subject: [PATCH 07/13] use --not instead of listing refs --- dist/index.js | 23 ++++++++++++----------- src/warpbuild/mirror-cache.ts | 23 ++++++++++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/dist/index.js b/dist/index.js index 3bbc325..9d945ed 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41988,14 +41988,17 @@ async function uploadBranchDelta(settings) { info(`Branch delta upload skipped (${grant.kind})`); return; } - const excludes = await baseRefExcludes(repoPath); - if (excludes.length === 0) { + if (!(await hasBaseRefs(repoPath))) { info('No base refs to diff against; branch delta skipped'); return; } const tmp = tempBundlePath('branch'); try { await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. await exec_exec('git', [ '-C', repoPath, @@ -42003,7 +42006,8 @@ async function uploadBranchDelta(settings) { 'create', tmp, UPLOAD_TIP_REF, - ...excludes + '--not', + `--glob=${BASE_REFNS}/*` ]); await httpPut(grant.url, tmp); info(`Uploaded branch delta for '${plan.refKey}'`); @@ -42015,15 +42019,12 @@ async function uploadBranchDelta(settings) { }); } } -// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle -// base-relative (and therefore order-free). -async function baseRefExcludes(repoPath) { +// Whether any base ref was seeded — the guard that keeps the delta base-relative. The +// base is excluded by glob (see uploadBranchDelta), so we only need existence here. +async function hasBaseRefs(repoPath) { let out = ''; - await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); - return out - .split('\n') - .map(s => s.trim()) - .filter(Boolean); + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out.trim().length > 0; } async function downloadTo(url, dest) { const res = await fetch(url, { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 14ddefe..35b3938 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -254,8 +254,7 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { return } - const excludes = await baseRefExcludes(repoPath) - if (excludes.length === 0) { + if (!(await hasBaseRefs(repoPath))) { core.info('No base refs to diff against; branch delta skipped') return } @@ -263,6 +262,10 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { const tmp = tempBundlePath('branch') try { await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. await exec.exec('git', [ '-C', repoPath, @@ -270,7 +273,8 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { 'create', tmp, UPLOAD_TIP_REF, - ...excludes + '--not', + `--glob=${BASE_REFNS}/*` ]) await httpPut(grant.url, tmp) core.info(`Uploaded branch delta for '${plan.refKey}'`) @@ -286,19 +290,16 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { } } -// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle -// base-relative (and therefore order-free). -async function baseRefExcludes(repoPath: string): Promise { +// Whether any base ref was seeded — the guard that keeps the delta base-relative. The +// base is excluded by glob (see uploadBranchDelta), so we only need existence here. +async function hasBaseRefs(repoPath: string): Promise { let out = '' await exec.exec( 'git', - ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], + ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} ) - return out - .split('\n') - .map(s => s.trim()) - .filter(Boolean) + return out.trim().length > 0 } async function downloadTo(url: string, dest: string): Promise { From 7a2ed3a02393dfc3942cde192f1703c0d200cc43 Mon Sep 17 00:00:00 2001 From: darshanime Date: Mon, 13 Jul 2026 14:09:25 +0530 Subject: [PATCH 08/13] dont push empty bundles --- dist/index.js | 83 +++++++++++++++++++++----------- src/warpbuild/mirror-cache.ts | 91 +++++++++++++++++++++++------------ 2 files changed, 116 insertions(+), 58 deletions(-) diff --git a/dist/index.js b/dist/index.js index 9d945ed..04eb13a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41983,42 +41983,69 @@ async function uploadBranchDelta(settings) { } const repoPath = settings.repositoryPath; const sha = settings.commit; - const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha); - if (grant.kind !== 'grant') { - info(`Branch delta upload skipped (${grant.kind})`); - return; - } - if (!(await hasBaseRefs(repoPath))) { - info('No base refs to diff against; branch delta skipped'); - return; - } - const tmp = tempBundlePath('branch'); + await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); try { - await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); - // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo - // has hundreds of base refs, and that many args overflows the Windows command-line - // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the - // same base-relative (order-free) delta. - await exec_exec('git', [ - '-C', - repoPath, - 'bundle', - 'create', - tmp, - UPLOAD_TIP_REF, - '--not', - `--glob=${BASE_REFNS}/*` - ]); - await httpPut(grant.url, tmp); - info(`Uploaded branch delta for '${plan.refKey}'`); + if (!(await hasBaseRefs(repoPath))) { + info('No base refs to diff against; branch delta skipped'); + return; + } + // If the tip is already in the base there is no delta to roll, and `git bundle + // create` errors on an empty range ("Refusing to create empty bundle"). Detect it + // first — and before taking the server lock, so an empty delta never blocks the + // other jobs racing the same branch. + if (await branchDeltaIsEmpty(repoPath)) { + info('Tip already contained in the base; no branch delta to upload'); + return; + } + const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha); + if (grant.kind !== 'grant') { + info(`Branch delta upload skipped (${grant.kind})`); + return; + } + const tmp = tempBundlePath('branch'); + try { + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. + await exec_exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ]); + await httpPut(grant.url, tmp); + info(`Uploaded branch delta for '${plan.refKey}'`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } } finally { - await external_fs_namespaceObject.promises.rm(tmp, { force: true }); await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], { ignoreReturnCode: true }); } } +// The tip's delta against the base is empty when the tip is already reachable from the +// base — nothing new to bundle. +async function branchDeltaIsEmpty(repoPath) { + let out = ''; + await exec_exec('git', [ + '-C', + repoPath, + 'rev-list', + '--count', + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out.trim() === '0'; +} // Whether any base ref was seeded — the guard that keeps the delta base-relative. The // base is excluded by glob (see uploadBranchDelta), so we only need existence here. async function hasBaseRefs(repoPath) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 35b3938..42d85c0 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -248,38 +248,49 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { const repoPath = settings.repositoryPath const sha = settings.commit - const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha) - if (grant.kind !== 'grant') { - core.info(`Branch delta upload skipped (${grant.kind})`) - return - } - - if (!(await hasBaseRefs(repoPath))) { - core.info('No base refs to diff against; branch delta skipped') - return - } - - const tmp = tempBundlePath('branch') + await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) try { - await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) - // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo - // has hundreds of base refs, and that many args overflows the Windows command-line - // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the - // same base-relative (order-free) delta. - await exec.exec('git', [ - '-C', - repoPath, - 'bundle', - 'create', - tmp, - UPLOAD_TIP_REF, - '--not', - `--glob=${BASE_REFNS}/*` - ]) - await httpPut(grant.url, tmp) - core.info(`Uploaded branch delta for '${plan.refKey}'`) + if (!(await hasBaseRefs(repoPath))) { + core.info('No base refs to diff against; branch delta skipped') + return + } + // If the tip is already in the base there is no delta to roll, and `git bundle + // create` errors on an empty range ("Refusing to create empty bundle"). Detect it + // first — and before taking the server lock, so an empty delta never blocks the + // other jobs racing the same branch. + if (await branchDeltaIsEmpty(repoPath)) { + core.info('Tip already contained in the base; no branch delta to upload') + return + } + + const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha) + if (grant.kind !== 'grant') { + core.info(`Branch delta upload skipped (${grant.kind})`) + return + } + + const tmp = tempBundlePath('branch') + try { + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. + await exec.exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ]) + await httpPut(grant.url, tmp) + core.info(`Uploaded branch delta for '${plan.refKey}'`) + } finally { + await fs.promises.rm(tmp, {force: true}) + } } finally { - await fs.promises.rm(tmp, {force: true}) await exec.exec( 'git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], @@ -290,6 +301,26 @@ async function uploadBranchDelta(settings: IGitSourceSettings): Promise { } } +// The tip's delta against the base is empty when the tip is already reachable from the +// base — nothing new to bundle. +async function branchDeltaIsEmpty(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + [ + '-C', + repoPath, + 'rev-list', + '--count', + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + return out.trim() === '0' +} + // Whether any base ref was seeded — the guard that keeps the delta base-relative. The // base is excluded by glob (see uploadBranchDelta), so we only need existence here. async function hasBaseRefs(repoPath: string): Promise { From 607bd1493305af54ddfe69815d41794beffb4df9 Mon Sep 17 00:00:00 2001 From: darshanime Date: Mon, 13 Jul 2026 17:19:38 +0530 Subject: [PATCH 09/13] remove lfs skip --- __test__/warpbuild-mirror.test.ts | 22 +++-- dist/index.js | 145 ++++++++++++++++++++++++++---- src/git-source-provider.ts | 42 ++++++--- src/warpbuild/mirror-cache.ts | 122 +++++++++++++++++++++++-- 4 files changed, 289 insertions(+), 42 deletions(-) diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index e6250e9..1d860f4 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -106,24 +106,34 @@ describe('warpbuild mirror cache', () => { ) }) - it('serves any fetch-depth and fetch-tags (mirror is full history)', () => { + it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => { expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull() - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain( + 'explicit shallow depth' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain( + 'explicit shallow depth' + ) expect( getMirrorCacheSkipReason(settingsFor({fetchTags: true})) ).toBeNull() }) - it('skips filters, sparse and lfs checkouts', () => { + it('skips filters and sparse checkouts', () => { expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( 'a fetch filter is configured' ) expect( getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) ).toBe('sparse checkout is configured') - expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( - 'lfs is enabled (lfs objects are not in the mirror)' - ) + }) + + it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => { + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull() + expect( + getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1})) + ).toBeNull() }) }) diff --git a/dist/index.js b/dist/index.js index 04eb13a..943b39f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41786,6 +41786,7 @@ async function requestBranchUpload(repoKey, ref, sha) { + // WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on // S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is // only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, @@ -41801,8 +41802,13 @@ const BASE_REFNS = 'refs/wb/base'; const BRANCH_REFNS = 'refs/wb/branch'; const UPLOAD_TIP_REF = 'refs/wb/tip'; let plan = { mode: 'off', repoKey: '', refKey: '' }; -// Null = attempt the cache; else a reason to log. The mirror serves full history for any -// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. function getMirrorCacheSkipReason(settings) { if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { @@ -41822,15 +41828,18 @@ function getMirrorCacheSkipReason(settings) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on'; } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`; + } if (settings.filter) { return 'a fetch filter is configured'; } if (settings.sparseCheckout) { return 'sparse checkout is configured'; } - if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the mirror)'; - } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. return null; } // The durable branch to key the per-branch bundle on. For pull_request events the merge @@ -41858,6 +41867,14 @@ async function setup(settings) { setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false'); return mode; } +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +async function abandon(settings) { + plan = { mode: 'off', repoKey: '', refKey: '' }; + setOutput('cache-hit', 'false'); + await cleanupWbRefs(settings.repositoryPath); +} async function setupImpl(settings) { plan = { mode: 'off', repoKey: '', refKey: '' }; const skipReason = getMirrorCacheSkipReason(settings); @@ -41871,6 +41888,9 @@ async function setupImpl(settings) { } catch (error) { warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath); return 'off'; } finally { @@ -41902,6 +41922,14 @@ async function setupInner(settings) { } // Restore: seed the base, then this branch's delta if present. await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS); + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath); + } if (lookup.branch) { try { await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS); @@ -41914,20 +41942,42 @@ async function setupInner(settings) { plan = { mode: 'seeded', repoKey, refKey }; return 'seeded'; } -// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) -// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. async function contribute(settings) { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return; + } + const guard = (error) => { + warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`); + process.exit(0); + }; + process.on('uncaughtException', guard); + process.on('unhandledRejection', guard); try { if (plan.mode === 'cold-build') { await uploadBaseMirror(settings); } - else if (plan.mode === 'seeded') { + else { await uploadBranchDelta(settings); } } catch (error) { warning(`WarpBuild mirror upload skipped: ${error}`); } + finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath); + } + process.removeListener('uncaughtException', guard); + process.removeListener('unhandledRejection', guard); + } } // Download a bundle and fetch its refs into refNs/* so its objects seed the local repo // (and count as "haves" for the delta negotiation). The bundle is a git-validated file; @@ -41950,6 +42000,44 @@ async function seedBundle(repoPath, url, refNs) { await external_fs_namespaceObject.promises.rm(tmp, { force: true }); } } +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath) { + let out = ''; + await exec_exec('git', [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); +} +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath) { + try { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); + } + catch (error) { + core_debug(`WarpBuild ref cleanup skipped: ${error}`); + } +} // Cold-build: after the forced full fetch the repo holds all branches under // refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would // also sweep up the ephemeral triggering PR merge ref) as the base and upload. @@ -42078,7 +42166,9 @@ async function httpPut(url, file) { } } function tempBundlePath(tag) { - return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`); + // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a + // shared tmpdir. + return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`); } // Kept for callers/tests: reset .git/objects to the empty `git init` shape. async function resetGitObjects(gitDir) { @@ -42224,15 +42314,28 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } + let warpbuildFetchDone = false; if (warpbuildMode === 'seeded') { // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded - // objects. No depth — the base is full history, so this stays non-shallow. - const refSpec = getRefSpec(settings.ref, settings.commit); - await git.fetch(refSpec, fetchOptions); - if (!(await testRef(git, settings.ref, settings.commit))) { - throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + - `The ref may have been updated after the workflow was triggered.`); + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = getRefSpec(settings.ref, settings.commit); + await git.fetch(refSpec, fetchOptions); + if (!(await testRef(git, settings.ref, settings.commit))) { + throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.`); + } + warpbuildFetchDone = true; } + catch (error) { + warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`); + await abandon(settings); + warpbuildMode = 'off'; + } + } + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags @@ -42300,8 +42403,6 @@ async function getSource(settings) { startGroup('Checking out the ref'); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); endGroup(); - // WarpBuild snapshot cache: upload the fetch result on a miss - await contribute(settings); // Submodules if (settings.submodules) { // Temporarily override global config @@ -42328,6 +42429,16 @@ async function getSource(settings) { setOutput('commit', commitSHA.trim()); // Check for incorrect pull request merge commit await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl); + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await contribute(settings); + } + catch (error) { + warning(`WarpBuild mirror upload skipped: ${error}`); + } } finally { // Remove auth diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index fbdf9d7..df892aa 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -190,17 +190,32 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } + let warpbuildFetchDone = false if (warpbuildMode === 'seeded') { // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded - // objects. No depth — the base is full history, so this stays non-shallow. - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { - throw new Error( - `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + - `The ref may have been updated after the workflow was triggered.` + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + throw new Error( + `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.` + ) + } + warpbuildFetchDone = true + } catch (error) { + core.warning( + `WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}` ) + await warpbuildMirror.abandon(settings) + warpbuildMode = 'off' } + } + + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( @@ -287,9 +302,6 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) core.endGroup() - // WarpBuild snapshot cache: upload the fetch result on a miss - await warpbuildMirror.contribute(settings) - // Submodules if (settings.submodules) { // Temporarily override global config @@ -332,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise { settings.commit, settings.githubServerUrl ) + + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await warpbuildMirror.contribute(settings) + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } } finally { // Remove auth if (authHelper) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 42d85c0..3ed93ad 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -2,6 +2,7 @@ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as io from '@actions/io' +import * as crypto from 'crypto' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' @@ -40,8 +41,13 @@ interface Plan { } let plan: Plan = {mode: 'off', repoKey: '', refKey: ''} -// Null = attempt the cache; else a reason to log. The mirror serves full history for any -// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { @@ -68,15 +74,18 @@ export function getMirrorCacheSkipReason( if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on' } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout` + } if (settings.filter) { return 'a fetch filter is configured' } if (settings.sparseCheckout) { return 'sparse checkout is configured' } - if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the mirror)' - } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. return null } @@ -107,6 +116,15 @@ export async function setup(settings: IGitSourceSettings): Promise { return mode } +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +export async function abandon(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} + core.setOutput('cache-hit', 'false') + await cleanupWbRefs(settings.repositoryPath) +} + async function setupImpl(settings: IGitSourceSettings): Promise { plan = {mode: 'off', repoKey: '', refKey: ''} const skipReason = getMirrorCacheSkipReason(settings) @@ -121,6 +139,9 @@ async function setupImpl(settings: IGitSourceSettings): Promise { core.warning( `WarpBuild mirror cache unavailable, using standard checkout: ${error}` ) + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath) return 'off' } finally { core.endGroup() @@ -159,6 +180,14 @@ async function setupInner(settings: IGitSourceSettings): Promise { // Restore: seed the base, then this branch's delta if present. await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS) + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath) + } if (lookup.branch) { try { await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS) @@ -175,17 +204,40 @@ async function setupInner(settings: IGitSourceSettings): Promise { return 'seeded' } -// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) -// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. export async function contribute(settings: IGitSourceSettings): Promise { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return + } + const guard = (error: unknown): void => { + core.warning( + `WarpBuild mirror upload error ignored — checkout already complete: ${error}` + ) + process.exit(0) + } + process.on('uncaughtException', guard) + process.on('unhandledRejection', guard) try { if (plan.mode === 'cold-build') { await uploadBaseMirror(settings) - } else if (plan.mode === 'seeded') { + } else { await uploadBranchDelta(settings) } } catch (error) { core.warning(`WarpBuild mirror upload skipped: ${error}`) + } finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath) + } + process.removeListener('uncaughtException', guard) + process.removeListener('unhandledRejection', guard) } } @@ -214,6 +266,53 @@ async function seedBundle( } } +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) +} + +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath: string): Promise { + try { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) + } catch (error) { + core.debug(`WarpBuild ref cleanup skipped: ${error}`) + } +} + // Cold-build: after the forced full fetch the repo holds all branches under // refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would // also sweep up the ephemeral triggering PR merge ref) as the base and upload. @@ -363,7 +462,12 @@ async function httpPut(url: string, file: string): Promise { } function tempBundlePath(tag: string): string { - return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`) + // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a + // shared tmpdir. + return path.join( + os.tmpdir(), + `wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle` + ) } // Kept for callers/tests: reset .git/objects to the empty `git init` shape. From 48babd5495d37408859426b489f98c7005e1a7aa Mon Sep 17 00:00:00 2001 From: darshanime Date: Tue, 14 Jul 2026 14:45:55 +0530 Subject: [PATCH 10/13] add ranged multiget --- dist/index.js | 82 ++++++++++++++++++++++++++++++++ src/warpbuild/mirror-cache.ts | 88 +++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/dist/index.js b/dist/index.js index 943b39f..a0d853c 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41794,6 +41794,12 @@ async function requestBranchUpload(repoKey, ref, sha) { // standard checkout. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Concurrent range-download tuning. A single stream saturates at the per-connection ceiling +// (~15 MB/s on a WAN); parallel range GETs reach link rate (~200 MB/s), as WarpBuilds/cache does. +const SEGMENT_SIZE = 8 * 1024 * 1024; +const SEGMENT_CONCURRENCY = 8; +const SEGMENT_TIMEOUT_MS = 60_000; +const SEGMENT_RETRIES = 5; // Logged at info when off; the WARPBUILD_* env is only present on our runners. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; @@ -42141,7 +42147,83 @@ async function hasBaseRefs(repoPath) { await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); return out.trim().length > 0; } +// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle +// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. +// Falls back to a single stream when the server doesn't support ranges. async function downloadTo(url, dest) { + const total = await probeRangeSize(url); + if (total === null) { + await downloadSingleStream(url, dest); + return; + } + const fh = await external_fs_namespaceObject.promises.open(dest, 'w'); + try { + const segments = []; + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); + } + let next = 0; + const worker = async () => { + for (;;) { + const i = next++; + if (i >= segments.length) { + return; + } + const [offset, count] = segments[i]; + const buf = await downloadSegment(url, offset, count); + await fh.write(buf, 0, count, offset); + } + }; + const pool = []; + for (let k = 0; k < Math.min(SEGMENT_CONCURRENCY, segments.length); k++) { + pool.push(worker()); + } + await Promise.all(pool); + } + finally { + await fh.close(); + } +} +// GET bytes=0-0 to learn the total size and confirm range support; null → not supported. +async function probeRangeSize(url) { + const res = await fetch(url, { + headers: { range: 'bytes=0-0' }, + signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + }); + if (res.status !== 206) { + return null; + } + const match = res.headers.get('content-range')?.match(/\/(\d+)\s*$/); + const total = match ? parseInt(match[1], 10) : NaN; + return Number.isFinite(total) && total > 0 ? total : null; +} +// One range GET with per-segment timeout + retries; returns exactly `count` bytes. +async function downloadSegment(url, offset, count) { + let lastErr; + for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { + try { + const res = await fetch(url, { + headers: { range: `bytes=${offset}-${offset + count - 1}` }, + signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + }); + if (res.status !== 206) { + throw new Error(`range GET answered ${res.status}`); + } + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length !== count) { + throw new Error(`short segment at ${offset}: ${buf.length}/${count}`); + } + return buf; + } + catch (error) { + lastErr = error; + } + } + throw lastErr instanceof Error + ? lastErr + : new Error(`segment at ${offset} failed`); +} +async function downloadSingleStream(url, dest) { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }); diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 3ed93ad..28ffdcc 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -20,6 +20,13 @@ import * as api from './backend-api.js' const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const UPLOAD_TIMEOUT_MS = 15 * 60_000 +// Concurrent range-download tuning. A single stream saturates at the per-connection ceiling +// (~15 MB/s on a WAN); parallel range GETs reach link rate (~200 MB/s), as WarpBuilds/cache does. +const SEGMENT_SIZE = 8 * 1024 * 1024 +const SEGMENT_CONCURRENCY = 8 +const SEGMENT_TIMEOUT_MS = 60_000 +const SEGMENT_RETRIES = 5 + // Logged at info when off; the WARPBUILD_* env is only present on our runners. export const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)' @@ -432,7 +439,88 @@ async function hasBaseRefs(repoPath: string): Promise { return out.trim().length > 0 } +// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle +// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. +// Falls back to a single stream when the server doesn't support ranges. async function downloadTo(url: string, dest: string): Promise { + const total = await probeRangeSize(url) + if (total === null) { + await downloadSingleStream(url, dest) + return + } + const fh = await fs.promises.open(dest, 'w') + try { + const segments: [number, number][] = [] + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) + } + let next = 0 + const worker = async (): Promise => { + for (;;) { + const i = next++ + if (i >= segments.length) { + return + } + const [offset, count] = segments[i] + const buf = await downloadSegment(url, offset, count) + await fh.write(buf, 0, count, offset) + } + } + const pool: Promise[] = [] + for (let k = 0; k < Math.min(SEGMENT_CONCURRENCY, segments.length); k++) { + pool.push(worker()) + } + await Promise.all(pool) + } finally { + await fh.close() + } +} + +// GET bytes=0-0 to learn the total size and confirm range support; null → not supported. +async function probeRangeSize(url: string): Promise { + const res = await fetch(url, { + headers: {range: 'bytes=0-0'}, + signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + }) + if (res.status !== 206) { + return null + } + const match = res.headers.get('content-range')?.match(/\/(\d+)\s*$/) + const total = match ? parseInt(match[1], 10) : NaN + return Number.isFinite(total) && total > 0 ? total : null +} + +// One range GET with per-segment timeout + retries; returns exactly `count` bytes. +async function downloadSegment( + url: string, + offset: number, + count: number +): Promise { + let lastErr: unknown + for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { + try { + const res = await fetch(url, { + headers: {range: `bytes=${offset}-${offset + count - 1}`}, + signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + }) + if (res.status !== 206) { + throw new Error(`range GET answered ${res.status}`) + } + const buf = Buffer.from(await res.arrayBuffer()) + if (buf.length !== count) { + throw new Error(`short segment at ${offset}: ${buf.length}/${count}`) + } + return buf + } catch (error) { + lastErr = error + } + } + throw lastErr instanceof Error + ? lastErr + : new Error(`segment at ${offset} failed`) +} + +async function downloadSingleStream(url: string, dest: string): Promise { const res = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }) From 8ffbcf05595758187e27e3d4038317f5e82dd29b Mon Sep 17 00:00:00 2001 From: darshanime Date: Tue, 14 Jul 2026 17:37:59 +0530 Subject: [PATCH 11/13] add o11y for multiget --- dist/index.js | 29 +++++++++++++++++++------- src/warpbuild/mirror-cache.ts | 38 ++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/dist/index.js b/dist/index.js index a0d853c..c4d1a7f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41990,8 +41990,9 @@ async function contribute(settings) { // nothing is extracted into .git directly. async function seedBundle(repoPath, url, refNs) { const tmp = tempBundlePath('seed'); + const what = refNs === BASE_REFNS ? 'base' : 'branch'; try { - await downloadTo(url, tmp); + await downloadTo(url, tmp, what); await exec_exec('git', [ '-C', repoPath, @@ -42150,18 +42151,22 @@ async function hasBaseRefs(repoPath) { // Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle // (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. // Falls back to a single stream when the server doesn't support ranges. -async function downloadTo(url, dest) { +async function downloadTo(url, dest, what) { + const started = Date.now(); const total = await probeRangeSize(url); if (total === null) { await downloadSingleStream(url, dest); + const bytes = (await external_fs_namespaceObject.promises.stat(dest)).size; + info(`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`); return; } + const segments = []; + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length); const fh = await external_fs_namespaceObject.promises.open(dest, 'w'); try { - const segments = []; - for (let off = 0; off < total; off += SEGMENT_SIZE) { - segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); - } let next = 0; const worker = async () => { for (;;) { @@ -42175,7 +42180,7 @@ async function downloadTo(url, dest) { } }; const pool = []; - for (let k = 0; k < Math.min(SEGMENT_CONCURRENCY, segments.length); k++) { + for (let k = 0; k < width; k++) { pool.push(worker()); } await Promise.all(pool); @@ -42183,6 +42188,16 @@ async function downloadTo(url, dest) { finally { await fh.close(); } + info(`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`); +} +// Size + elapsed/throughput for the mirror download o11y line. +function fmtSize(bytes) { + return `${(bytes / 1e6).toFixed(1)} MB`; +} +function fmtElapsed(startedMs, bytes) { + const secs = (Date.now() - startedMs) / 1000; + const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a'; + return `${secs.toFixed(2)}s (${rate} MB/s)`; } // GET bytes=0-0 to learn the total size and confirm range support; null → not supported. async function probeRangeSize(url) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 28ffdcc..c2bbfac 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -257,8 +257,9 @@ async function seedBundle( refNs: string ): Promise { const tmp = tempBundlePath('seed') + const what = refNs === BASE_REFNS ? 'base' : 'branch' try { - await downloadTo(url, tmp) + await downloadTo(url, tmp, what) await exec.exec('git', [ '-C', repoPath, @@ -442,18 +443,28 @@ async function hasBaseRefs(repoPath: string): Promise { // Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle // (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. // Falls back to a single stream when the server doesn't support ranges. -async function downloadTo(url: string, dest: string): Promise { +async function downloadTo( + url: string, + dest: string, + what: string +): Promise { + const started = Date.now() const total = await probeRangeSize(url) if (total === null) { await downloadSingleStream(url, dest) + const bytes = (await fs.promises.stat(dest)).size + core.info( + `Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}` + ) return } + const segments: [number, number][] = [] + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length) const fh = await fs.promises.open(dest, 'w') try { - const segments: [number, number][] = [] - for (let off = 0; off < total; off += SEGMENT_SIZE) { - segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) - } let next = 0 const worker = async (): Promise => { for (;;) { @@ -467,13 +478,26 @@ async function downloadTo(url: string, dest: string): Promise { } } const pool: Promise[] = [] - for (let k = 0; k < Math.min(SEGMENT_CONCURRENCY, segments.length); k++) { + for (let k = 0; k < width; k++) { pool.push(worker()) } await Promise.all(pool) } finally { await fh.close() } + core.info( + `Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}` + ) +} + +// Size + elapsed/throughput for the mirror download o11y line. +function fmtSize(bytes: number): string { + return `${(bytes / 1e6).toFixed(1)} MB` +} +function fmtElapsed(startedMs: number, bytes: number): string { + const secs = (Date.now() - startedMs) / 1000 + const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a' + return `${secs.toFixed(2)}s (${rate} MB/s)` } // GET bytes=0-0 to learn the total size and confirm range support; null → not supported. From d6005557e6a4d7d293ba24d35f51b26b5b52c601 Mon Sep 17 00:00:00 2001 From: darshanime Date: Tue, 14 Jul 2026 18:05:45 +0530 Subject: [PATCH 12/13] use @actions/http-client --- dist/index.js | 205 +++++++++++++++++++++++----------- package.json | 1 + src/warpbuild/mirror-cache.ts | 144 +++++++++++++----------- 3 files changed, 221 insertions(+), 129 deletions(-) diff --git a/dist/index.js b/dist/index.js index c4d1a7f..49a5c0d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -31883,6 +31883,64 @@ function qstring(str) { /******/ } /******/ /************************************************************************/ +/******/ /* webpack/runtime/create fake namespace object */ +/******/ (() => { +/******/ var getProto = Object.getPrototypeOf ? (obj) => (Object.getPrototypeOf(obj)) : (obj) => (obj.__proto__); +/******/ var leafPrototypes; +/******/ // create a fake namespace object +/******/ // mode & 1: value is a module id, require it +/******/ // mode & 2: merge all properties of value into the ns +/******/ // mode & 4: return value when already ns object +/******/ // mode & 16: return value when it's Promise-like +/******/ // mode & 8|1: behave like require +/******/ __nccwpck_require__.t = function(value, mode) { +/******/ if(mode & 1) value = this(value); +/******/ if(mode & 8) return value; +/******/ if(typeof value === 'object' && value) { +/******/ if((mode & 4) && value.__esModule) return value; +/******/ if((mode & 16) && typeof value.then === 'function') return value; +/******/ } +/******/ var ns = Object.create(null); +/******/ __nccwpck_require__.r(ns); +/******/ var def = {}; +/******/ leafPrototypes = leafPrototypes || [null, getProto({}), getProto([]), getProto(getProto)]; +/******/ for(var current = mode & 2 && value; typeof current == 'object' && !~leafPrototypes.indexOf(current); current = getProto(current)) { +/******/ Object.getOwnPropertyNames(current).forEach((key) => (def[key] = () => (value[key]))); +/******/ } +/******/ def['default'] = () => (value); +/******/ __nccwpck_require__.d(ns, def); +/******/ return ns; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __nccwpck_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __nccwpck_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ /******/ /* webpack/runtime/compat */ /******/ /******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/"; @@ -32063,8 +32121,10 @@ function file_command_prepareKeyValueMessage(key, value) { const external_path_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("path"); // EXTERNAL MODULE: external "http" var external_http_ = __nccwpck_require__(8611); +var external_http_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_http_, 2); // EXTERNAL MODULE: external "https" var external_https_ = __nccwpck_require__(5692); +var external_https_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_https_, 2); ;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/proxy.js function getProxyUrl(reqUrl) { const usingSsl = reqUrl.protocol === 'https:'; @@ -32157,7 +32217,7 @@ class DecodedURL extends URL { } //# sourceMappingURL=proxy.js.map // EXTERNAL MODULE: ./node_modules/tunnel/index.js -var node_modules_tunnel = __nccwpck_require__(770); +var tunnel = __nccwpck_require__(770); // EXTERNAL MODULE: ./node_modules/undici/index.js var undici = __nccwpck_require__(6752); ;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/index.js @@ -32235,7 +32295,7 @@ const HttpResponseRetryCodes = [ HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout ]; -const RetryableHttpVerbs = (/* unused pure expression or super */ null && (['OPTIONS', 'GET', 'DELETE', 'HEAD'])); +const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; class HttpClientError extends Error { @@ -32584,7 +32644,7 @@ class lib_HttpClient { } getAgentDispatcher(serverUrl) { const parsedUrl = new URL(serverUrl); - const proxyUrl = pm.getProxyUrl(parsedUrl); + const proxyUrl = getProxyUrl(parsedUrl); const useProxy = proxyUrl && proxyUrl.hostname; if (!useProxy) { return; @@ -32595,7 +32655,7 @@ class lib_HttpClient { const info = {}; info.parsedUrl = requestUrl; const usingSsl = info.parsedUrl.protocol === 'https:'; - info.httpModule = usingSsl ? https : http; + info.httpModule = usingSsl ? external_https_namespaceObject : external_http_namespaceObject; const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; @@ -32694,7 +32754,7 @@ class lib_HttpClient { } _getAgent(parsedUrl) { let agent; - const proxyUrl = pm.getProxyUrl(parsedUrl); + const proxyUrl = getProxyUrl(parsedUrl); const useProxy = proxyUrl && proxyUrl.hostname; if (this._keepAlive && useProxy) { agent = this._proxyAgent; @@ -32709,7 +32769,7 @@ class lib_HttpClient { const usingSsl = parsedUrl.protocol === 'https:'; let maxSockets = 100; if (this.requestOptions) { - maxSockets = this.requestOptions.maxSockets || http.globalAgent.maxSockets; + maxSockets = this.requestOptions.maxSockets || external_http_.globalAgent.maxSockets; } // This is `useProxy` again, but we need to check `proxyURl` directly for TypeScripts's flow analysis. if (proxyUrl && proxyUrl.hostname) { @@ -32734,7 +32794,7 @@ class lib_HttpClient { // if tunneling agent isn't assigned create a new agent if (!agent) { const options = { keepAlive: this._keepAlive, maxSockets }; - agent = usingSsl ? new https.Agent(options) : new http.Agent(options); + agent = usingSsl ? new external_https_.Agent(options) : new external_http_.Agent(options); this._agent = agent; } if (usingSsl && this._ignoreSslError) { @@ -32757,7 +32817,7 @@ class lib_HttpClient { return proxyAgent; } const usingSsl = parsedUrl.protocol === 'https:'; - proxyAgent = new ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && { + proxyAgent = new undici.ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && { token: `Basic ${Buffer.from(`${proxyUrl.username}:${proxyUrl.password}`).toString('base64')}` }))); this._proxyAgentDispatcher = proxyAgent; @@ -41792,13 +41852,13 @@ async function requestBranchUpload(repoKey, ref, sha) { // only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, // so order-free); a cold repo single-flights the base build. Fail-open: any error → // standard checkout. -const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const UPLOAD_TIMEOUT_MS = 15 * 60_000; -// Concurrent range-download tuning. A single stream saturates at the per-connection ceiling -// (~15 MB/s on a WAN); parallel range GETs reach link rate (~200 MB/s), as WarpBuilds/cache does. -const SEGMENT_SIZE = 8 * 1024 * 1024; +// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a +// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a +// WAN); parallel range GETs reach link rate (~200 MB/s). +const SEGMENT_SIZE = 4 * 1024 * 1024; const SEGMENT_CONCURRENCY = 8; -const SEGMENT_TIMEOUT_MS = 60_000; +const SEGMENT_TIMEOUT_MS = 30_000; const SEGMENT_RETRIES = 5; // Logged at info when off; the WARPBUILD_* env is only present on our runners. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; @@ -42153,42 +42213,52 @@ async function hasBaseRefs(repoPath) { // Falls back to a single stream when the server doesn't support ranges. async function downloadTo(url, dest, what) { const started = Date.now(); - const total = await probeRangeSize(url); - if (total === null) { - await downloadSingleStream(url, dest); - const bytes = (await external_fs_namespaceObject.promises.stat(dest)).size; - info(`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`); - return; - } - const segments = []; - for (let off = 0; off < total; off += SEGMENT_SIZE) { - segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); - } - const width = Math.min(SEGMENT_CONCURRENCY, segments.length); - const fh = await external_fs_namespaceObject.promises.open(dest, 'w'); + // Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes. + const http = new lib_HttpClient('warpbuild-checkout-mirror', undefined, { + socketTimeout: SEGMENT_TIMEOUT_MS, + keepAlive: true + }); try { - let next = 0; - const worker = async () => { - for (;;) { - const i = next++; - if (i >= segments.length) { - return; - } - const [offset, count] = segments[i]; - const buf = await downloadSegment(url, offset, count); - await fh.write(buf, 0, count, offset); - } - }; - const pool = []; - for (let k = 0; k < width; k++) { - pool.push(worker()); + const total = await probeRangeSize(http, url); + if (total === null) { + await downloadSingleStream(http, url, dest); + const bytes = (await external_fs_namespaceObject.promises.stat(dest)).size; + info(`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`); + return; } - await Promise.all(pool); + const segments = []; + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length); + const fh = await external_fs_namespaceObject.promises.open(dest, 'w'); + try { + let next = 0; + const worker = async () => { + for (;;) { + const i = next++; + if (i >= segments.length) { + return; + } + const [offset, count] = segments[i]; + const buf = await downloadSegment(http, url, offset, count); + await fh.write(buf, 0, count, offset); + } + }; + const pool = []; + for (let k = 0; k < width; k++) { + pool.push(worker()); + } + await Promise.all(pool); + } + finally { + await fh.close(); + } + info(`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`); } finally { - await fh.close(); + http.dispose(); } - info(`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`); } // Size + elapsed/throughput for the mirror download o11y line. function fmtSize(bytes) { @@ -42200,31 +42270,34 @@ function fmtElapsed(startedMs, bytes) { return `${secs.toFixed(2)}s (${rate} MB/s)`; } // GET bytes=0-0 to learn the total size and confirm range support; null → not supported. -async function probeRangeSize(url) { - const res = await fetch(url, { - headers: { range: 'bytes=0-0' }, - signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) - }); - if (res.status !== 206) { +async function probeRangeSize(http, url) { + const res = await http.get(url, { Range: 'bytes=0-0' }); + if (res.message.statusCode !== 206) { + res.message.destroy(); // no range support; don't read a full 200 body into memory return null; } - const match = res.headers.get('content-range')?.match(/\/(\d+)\s*$/); + await res.readBody(); // 1 byte; return the socket to the keep-alive pool + const contentRange = res.message.headers['content-range']; + const match = typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null; const total = match ? parseInt(match[1], 10) : NaN; return Number.isFinite(total) && total > 0 ? total : null; } -// One range GET with per-segment timeout + retries; returns exactly `count` bytes. -async function downloadSegment(url, offset, count) { +// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes. +async function downloadSegment(http, url, offset, count) { let lastErr; for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { try { - const res = await fetch(url, { - headers: { range: `bytes=${offset}-${offset + count - 1}` }, - signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + const res = await http.get(url, { + Range: `bytes=${offset}-${offset + count - 1}` }); - if (res.status !== 206) { - throw new Error(`range GET answered ${res.status}`); + if (res.message.statusCode !== 206) { + await res.readBody(); // drain before retrying so the socket is reusable + throw new Error(`range GET answered ${res.message.statusCode}`); + } + const buf = await res.readBodyBuffer?.(); + if (!buf) { + throw new Error('http-client response lacks readBodyBuffer'); } - const buf = Buffer.from(await res.arrayBuffer()); if (buf.length !== count) { throw new Error(`short segment at ${offset}: ${buf.length}/${count}`); } @@ -42238,14 +42311,14 @@ async function downloadSegment(url, offset, count) { ? lastErr : new Error(`segment at ${offset} failed`); } -async function downloadSingleStream(url, dest) { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }); - if (!res.ok || !res.body) { - throw new Error(`download answered ${res.status}`); +async function downloadSingleStream(http, url, dest) { + const res = await http.get(url); + const status = res.message.statusCode ?? 0; + if (status < 200 || status >= 300) { + await res.readBody(); + throw new Error(`download answered ${status}`); } - await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(dest)); + await (0,promises_namespaceObject.pipeline)(res.message, external_fs_namespaceObject.createWriteStream(dest)); } async function httpPut(url, file) { const stat = await external_fs_namespaceObject.promises.stat(file); diff --git a/package.json b/package.json index 594de95..a379b5e 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@actions/core": "^3.0.1", "@actions/exec": "^3.0.0", "@actions/github": "^9.1.1", + "@actions/http-client": "^4.0.1", "@actions/io": "^3.0.2", "@actions/tool-cache": "^4.0.0" }, diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index c2bbfac..fdb9308 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -2,11 +2,11 @@ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as io from '@actions/io' +import {HttpClient} from '@actions/http-client' import * as crypto from 'crypto' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' -import {Readable} from 'stream' import {pipeline} from 'stream/promises' import {IGitSourceSettings} from '../git-source-settings.js' import * as api from './backend-api.js' @@ -17,14 +17,14 @@ import * as api from './backend-api.js' // so order-free); a cold repo single-flights the base build. Fail-open: any error → // standard checkout. -const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const UPLOAD_TIMEOUT_MS = 15 * 60_000 -// Concurrent range-download tuning. A single stream saturates at the per-connection ceiling -// (~15 MB/s on a WAN); parallel range GETs reach link rate (~200 MB/s), as WarpBuilds/cache does. -const SEGMENT_SIZE = 8 * 1024 * 1024 +// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a +// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a +// WAN); parallel range GETs reach link rate (~200 MB/s). +const SEGMENT_SIZE = 4 * 1024 * 1024 const SEGMENT_CONCURRENCY = 8 -const SEGMENT_TIMEOUT_MS = 60_000 +const SEGMENT_TIMEOUT_MS = 30_000 const SEGMENT_RETRIES = 5 // Logged at info when off; the WARPBUILD_* env is only present on our runners. @@ -449,45 +449,54 @@ async function downloadTo( what: string ): Promise { const started = Date.now() - const total = await probeRangeSize(url) - if (total === null) { - await downloadSingleStream(url, dest) - const bytes = (await fs.promises.stat(dest)).size - core.info( - `Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}` - ) - return - } - const segments: [number, number][] = [] - for (let off = 0; off < total; off += SEGMENT_SIZE) { - segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) - } - const width = Math.min(SEGMENT_CONCURRENCY, segments.length) - const fh = await fs.promises.open(dest, 'w') + // Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes. + const http = new HttpClient('warpbuild-checkout-mirror', undefined, { + socketTimeout: SEGMENT_TIMEOUT_MS, + keepAlive: true + }) try { - let next = 0 - const worker = async (): Promise => { - for (;;) { - const i = next++ - if (i >= segments.length) { - return + const total = await probeRangeSize(http, url) + if (total === null) { + await downloadSingleStream(http, url, dest) + const bytes = (await fs.promises.stat(dest)).size + core.info( + `Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}` + ) + return + } + const segments: [number, number][] = [] + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length) + const fh = await fs.promises.open(dest, 'w') + try { + let next = 0 + const worker = async (): Promise => { + for (;;) { + const i = next++ + if (i >= segments.length) { + return + } + const [offset, count] = segments[i] + const buf = await downloadSegment(http, url, offset, count) + await fh.write(buf, 0, count, offset) } - const [offset, count] = segments[i] - const buf = await downloadSegment(url, offset, count) - await fh.write(buf, 0, count, offset) } + const pool: Promise[] = [] + for (let k = 0; k < width; k++) { + pool.push(worker()) + } + await Promise.all(pool) + } finally { + await fh.close() } - const pool: Promise[] = [] - for (let k = 0; k < width; k++) { - pool.push(worker()) - } - await Promise.all(pool) + core.info( + `Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}` + ) } finally { - await fh.close() + http.dispose() } - core.info( - `Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}` - ) } // Size + elapsed/throughput for the mirror download o11y line. @@ -501,21 +510,26 @@ function fmtElapsed(startedMs: number, bytes: number): string { } // GET bytes=0-0 to learn the total size and confirm range support; null → not supported. -async function probeRangeSize(url: string): Promise { - const res = await fetch(url, { - headers: {range: 'bytes=0-0'}, - signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) - }) - if (res.status !== 206) { +async function probeRangeSize( + http: HttpClient, + url: string +): Promise { + const res = await http.get(url, {Range: 'bytes=0-0'}) + if (res.message.statusCode !== 206) { + res.message.destroy() // no range support; don't read a full 200 body into memory return null } - const match = res.headers.get('content-range')?.match(/\/(\d+)\s*$/) + await res.readBody() // 1 byte; return the socket to the keep-alive pool + const contentRange = res.message.headers['content-range'] + const match = + typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null const total = match ? parseInt(match[1], 10) : NaN return Number.isFinite(total) && total > 0 ? total : null } -// One range GET with per-segment timeout + retries; returns exactly `count` bytes. +// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes. async function downloadSegment( + http: HttpClient, url: string, offset: number, count: number @@ -523,14 +537,17 @@ async function downloadSegment( let lastErr: unknown for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { try { - const res = await fetch(url, { - headers: {range: `bytes=${offset}-${offset + count - 1}`}, - signal: AbortSignal.timeout(SEGMENT_TIMEOUT_MS) + const res = await http.get(url, { + Range: `bytes=${offset}-${offset + count - 1}` }) - if (res.status !== 206) { - throw new Error(`range GET answered ${res.status}`) + if (res.message.statusCode !== 206) { + await res.readBody() // drain before retrying so the socket is reusable + throw new Error(`range GET answered ${res.message.statusCode}`) + } + const buf = await res.readBodyBuffer?.() + if (!buf) { + throw new Error('http-client response lacks readBodyBuffer') } - const buf = Buffer.from(await res.arrayBuffer()) if (buf.length !== count) { throw new Error(`short segment at ${offset}: ${buf.length}/${count}`) } @@ -544,17 +561,18 @@ async function downloadSegment( : new Error(`segment at ${offset} failed`) } -async function downloadSingleStream(url: string, dest: string): Promise { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }) - if (!res.ok || !res.body) { - throw new Error(`download answered ${res.status}`) +async function downloadSingleStream( + http: HttpClient, + url: string, + dest: string +): Promise { + const res = await http.get(url) + const status = res.message.statusCode ?? 0 + if (status < 200 || status >= 300) { + await res.readBody() + throw new Error(`download answered ${status}`) } - await pipeline( - Readable.fromWeb(res.body as import('stream/web').ReadableStream), - fs.createWriteStream(dest) - ) + await pipeline(res.message, fs.createWriteStream(dest)) } async function httpPut(url: string, file: string): Promise { From 28b21c7d05cbaf15e2e7c1ff5941ad44f831c4aa Mon Sep 17 00:00:00 2001 From: darshanime Date: Tue, 14 Jul 2026 18:25:02 +0530 Subject: [PATCH 13/13] use mkdtempSync --- dist/index.js | 10 +++++++--- src/warpbuild/mirror-cache.ts | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dist/index.js b/dist/index.js index 49a5c0d..5c58625 100644 --- a/dist/index.js +++ b/dist/index.js @@ -42335,10 +42335,14 @@ async function httpPut(url, file) { throw new Error(`PUT answered ${res.status}`); } } +// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a +// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file). +let mirrorTmpDir; function tempBundlePath(tag) { - // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a - // shared tmpdir. - return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`); + if (!mirrorTmpDir) { + mirrorTmpDir = external_fs_namespaceObject.mkdtempSync(external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), 'wb-mirror-')); + } + return external_path_namespaceObject.join(mirrorTmpDir, `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`); } // Kept for callers/tests: reset .git/objects to the empty `git init` shape. async function resetGitObjects(gitDir) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index fdb9308..8d2953b 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -591,11 +591,15 @@ async function httpPut(url: string, file: string): Promise { } } +// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a +// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file). +let mirrorTmpDir: string | undefined function tempBundlePath(tag: string): string { - // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a - // shared tmpdir. + if (!mirrorTmpDir) { + mirrorTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wb-mirror-')) + } return path.join( - os.tmpdir(), + mirrorTmpDir, `wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle` ) }