diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index 7d280fd..e6250e9 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -6,26 +6,29 @@ import { afterEach, afterAll } from '@jest/globals' -import * as exec from '@actions/exec' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' import { SKIP_NOT_WARPBUILD, - assertSafeTarMembers, - computeDestinationRef, + computeRefKey, getMirrorCacheSkipReason, - manifestDir, resetGitObjects } from '../src/warpbuild/mirror-cache.js' -import {lookupSnapshot} from '../src/warpbuild/backend-api.js' +import { + lookupRestore, + requestBaseUpload, + requestBranchUpload +} from '../src/warpbuild/backend-api.js' import {IGitSourceSettings} from '../src/git-source-settings.js' const WB_ENV = [ 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', 'WARPBUILD_HOST_URL', 'GITHUB_REPOSITORY_ID', - 'GITHUB_REPOSITORY' + 'GITHUB_REPOSITORY', + 'GITHUB_REF', + 'GITHUB_BASE_REF' ] const savedEnv: {[key: string]: string | undefined} = {} for (const key of WB_ENV) { @@ -57,9 +60,11 @@ function setWarpBuildEnv(): void { process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' process.env['GITHUB_REPOSITORY_ID'] = '123456789' process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' + delete process.env['GITHUB_REF'] + delete process.env['GITHUB_BASE_REF'] } -describe('warpbuild snapshot cache', () => { +describe('warpbuild mirror cache', () => { beforeEach(() => { setWarpBuildEnv() }) @@ -75,9 +80,7 @@ describe('warpbuild snapshot cache', () => { }) describe('getMirrorCacheSkipReason', () => { - const winOnly = process.platform === 'win32' ? it.skip : it - - winOnly('returns null for the default checkout shape', () => { + it('returns null for the default checkout shape', () => { expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() }) @@ -86,7 +89,7 @@ describe('warpbuild snapshot cache', () => { expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) }) - it('reports repository: inputs that are not the workflow repo', () => { + it('reports repository inputs that are not the workflow repo', () => { expect( getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) ).toBe( @@ -103,19 +106,15 @@ describe('warpbuild snapshot cache', () => { ) }) - it('only serves fetch-depth 1', () => { - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( - 'fetch-depth is 0, cache only serves fetch-depth 1' - ) - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( - 'fetch-depth is 50, cache only serves fetch-depth 1' - ) + it('serves any fetch-depth and fetch-tags (mirror is full history)', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull() + expect( + getMirrorCacheSkipReason(settingsFor({fetchTags: true})) + ).toBeNull() }) - it('skips fetch-tags, filters, sparse and lfs checkouts', () => { - expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe( - 'fetch-tags is enabled' - ) + it('skips filters, sparse and lfs checkouts', () => { expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( 'a fetch filter is configured' ) @@ -123,18 +122,36 @@ describe('warpbuild snapshot cache', () => { getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) ).toBe('sparse checkout is configured') expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( - 'lfs is enabled (lfs objects are not in the snapshot)' + 'lfs is enabled (lfs objects are not in the mirror)' ) }) + }) - winOnly('accepts sha-only checkouts (empty ref)', () => { - expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() + describe('computeRefKey', () => { + it('keys on the pushed branch (GITHUB_REF)', () => { + process.env['GITHUB_REF'] = 'refs/heads/feature/x' + expect(computeRefKey(settingsFor())).toBe('feature/x') }) - it('skips unqualified refs', () => { - expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( - "ref 'main' has no cacheable destination ref" - ) + it('keys on the base branch for pull requests (merge sha is synthetic)', () => { + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + process.env['GITHUB_BASE_REF'] = 'main' + expect(computeRefKey(settingsFor())).toBe('main') + }) + + it('is empty for tags and bare pull refs (base only, no roll)', () => { + process.env['GITHUB_REF'] = 'refs/tags/v1.2.3' + expect(computeRefKey(settingsFor())).toBe('') + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + expect(computeRefKey(settingsFor())).toBe('') + }) + + it('is empty for a detached sha checkout', () => { + expect(computeRefKey(settingsFor({ref: SHA}))).toBe('') + }) + + it('passes through an already-short branch name', () => { + expect(computeRefKey(settingsFor({ref: 'main'}))).toBe('main') }) }) @@ -144,7 +161,6 @@ describe('warpbuild snapshot cache', () => { path.join(os.tmpdir(), 'wb-reset-') ) try { - // Simulate a partially-restored .git: stray objects, a pack, a shallow file. await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { recursive: true }) @@ -152,179 +168,95 @@ describe('warpbuild snapshot cache', () => { path.join(gitDir, 'objects', 'ab'), 'junk loose object' ) - await fs.promises.writeFile( - path.join(gitDir, 'objects', 'pack', 'pack-x.pack'), - 'junk pack' - ) await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n') await resetGitObjects(gitDir) - // shallow gone; objects/ holds only the two empty init dirs. expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false) expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false) expect( (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() ).toEqual(['info', 'pack']) - expect( - await fs.promises.readdir(path.join(gitDir, 'objects', 'pack')) - ).toEqual([]) - expect( - await fs.promises.readdir(path.join(gitDir, 'objects', 'info')) - ).toEqual([]) } finally { await fs.promises.rm(gitDir, {recursive: true, force: true}) } }) - - it('is a no-op-safe on an already-clean .git', async () => { - const gitDir = await fs.promises.mkdtemp( - path.join(os.tmpdir(), 'wb-reset-') - ) - try { - await expect(resetGitObjects(gitDir)).resolves.toBeUndefined() - expect( - (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() - ).toEqual(['info', 'pack']) - } finally { - await fs.promises.rm(gitDir, {recursive: true, force: true}) - } - }) - }) - - describe('assertSafeTarMembers', () => { - async function makeTar( - dir: string, - members: string[], - writeFiles = true - ): Promise { - if (writeFiles) { - for (const m of members) { - const abs = path.join(dir, m) - await fs.promises.mkdir(path.dirname(abs), {recursive: true}) - await fs.promises.writeFile(abs, 'x') - } - } - const tar = path.join(dir, 'out.tar') - await exec.exec('tar', ['-cf', tar, '-C', dir, ...members]) - return tar - } - - it('accepts objects/ and shallow members', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow']) - await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined() - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - - it('rejects members outside objects/ and shallow (e.g. hooks)', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit']) - await expect(assertSafeTarMembers(tar)).rejects.toThrow( - /unexpected snapshot tar member/ - ) - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - - it('rejects path traversal members', async () => { - const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-')) - try { - await fs.promises.mkdir(path.join(dir, 'sub')) - const tar = path.join(dir, 'evil.tar') - // -P preserves the ../ member instead of stripping it. - await fs.promises.writeFile(path.join(dir, 'evil'), 'x') - await exec.exec('tar', [ - '-cPf', - tar, - '-C', - path.join(dir, 'sub'), - '../evil' - ]) - await expect(assertSafeTarMembers(tar)).rejects.toThrow( - /unexpected snapshot tar member/ - ) - } finally { - await fs.promises.rm(dir, {recursive: true, force: true}) - } - }) - }) - - describe('computeDestinationRef', () => { - it('maps the refs the fetch would have created', () => { - expect(computeDestinationRef('refs/heads/main')).toBe( - 'refs/remotes/origin/main' - ) - expect(computeDestinationRef('refs/pull/42/merge')).toBe( - 'refs/remotes/pull/42/merge' - ) - expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3') - expect(computeDestinationRef('')).toBe('') - expect(computeDestinationRef('main')).toBeNull() - }) }) describe('backend api http contract', () => { const realFetch = globalThis.fetch let lastUrl = '' + let lastInit: RequestInit | undefined afterEach(() => { globalThis.fetch = realFetch }) function stubFetch(status: number, body: unknown): void { - globalThis.fetch = (async (input: unknown) => { + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { lastUrl = String(input) + lastInit = init return new Response(JSON.stringify(body), {status}) }) as typeof fetch } - it('maps 200 to hit and keys by sha', async () => { - stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) - const result = await lookupSnapshot('123', SHA) - expect(result.kind).toBe('hit') - expect(lastUrl).toContain(`sha=${SHA}`) + it('lookupRestore maps 200 to a restore plan keyed by ref', async () => { + stubFetch(200, { + base: {url: 'https://s3/base', size_bytes: 42}, + branch: {url: 'https://s3/branch'} + }) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.base.url).toBe('https://s3/base') + expect(result.branch?.url).toBe('https://s3/branch') + } + expect(lastUrl).toContain('ref=main') }) - it('maps 404 to miss (agent uploads after the job)', async () => { - stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') - }) - - it('maps 403 to disabled (backend kill switch)', async () => { - stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') - }) - - it('maps other statuses and network failures to error', async () => { - stubFetch(500, {}) - expect((await lookupSnapshot('123', SHA)).kind).toBe('error') - globalThis.fetch = (async () => { - throw new Error('boom') - }) as typeof fetch - expect((await lookupSnapshot('123', SHA)).kind).toBe('error') - }) - }) - - describe('manifestDir', () => { - const saved = process.env['RUNNER_TEMP'] - afterEach(() => { - if (saved === undefined) { - delete process.env['RUNNER_TEMP'] - } else { - process.env['RUNNER_TEMP'] = saved + it('lookupRestore maps a null branch (base but no delta yet)', async () => { + stubFetch(200, {base: {url: 'https://s3/base'}, branch: null}) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.branch).toBeNull() } }) - it('places manifests under RUNNER_TEMP/wb-snapshots', () => { - process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x') - expect(manifestDir()).toBe( - path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots') + it('lookupRestore maps 404 to cold, 403 to disabled, 500 to error', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupRestore('123', 'main')).kind).toBe('cold') + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupRestore('123', 'main')).kind).toBe('disabled') + stubFetch(500, {}) + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('lookupRestore maps network failure to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('requestBaseUpload maps 200 to a grant, 409 to locked', async () => { + stubFetch(200, {url: 'https://s3/put-base'}) + const grant = await requestBaseUpload('123') + expect(grant.kind).toBe('grant') + if (grant.kind === 'grant') { + expect(grant.url).toBe('https://s3/put-base') + } + expect(lastInit?.method).toBe('POST') + stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) + expect((await requestBaseUpload('123')).kind).toBe('locked') + }) + + it('requestBranchUpload maps 200 to a grant, 403 to disabled', async () => { + stubFetch(200, {url: 'https://s3/put-branch'}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe('grant') + stubFetch(403, {}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe( + 'disabled' ) }) }) diff --git a/dist/index.js b/dist/index.js index 0a42bda..3bbc325 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41697,9 +41697,11 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts /* eslint-disable i18n-text/no-en -- upstream convention */ -// Client for backend-core's git-mirror download-url endpoint, authed by the runner -// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable -// org; else = fall back. The upload half runs in the warpbuild-agent after the job. +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. const API_TIMEOUT_MS = 10_000; function backend_api_baseUrl() { return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); @@ -41707,30 +41709,71 @@ function backend_api_baseUrl() { function authHeader() { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; } -async function lookupSnapshot(repoKey, sha) { +function backend_api_endpoint(p) { + return `${backend_api_baseUrl()}/api/v1/git-mirrors/${p}`; +} +async function lookupRestore(repoKey, ref) { try { - const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { + const res = await fetch(`${backend_api_endpoint('restore-url')}?repo_key=${encodeURIComponent(repoKey)}&ref=${encodeURIComponent(ref)}`, { headers: { authorization: authHeader() }, signal: AbortSignal.timeout(API_TIMEOUT_MS) }); if (res.status === 200) { - return { kind: 'hit', info: (await res.json()) }; + const body = (await res.json()); + return { kind: 'restore', base: body.base, branch: body.branch ?? null }; } if (res.status === 404) { - return { kind: 'miss' }; + return { kind: 'cold' }; } if (res.status === 403) { - core_debug(`[wb-cache] download-url answered 403 (disabled)`); + core_debug('[wb-cache] restore-url answered 403 (disabled)'); return { kind: 'disabled' }; } - core_debug(`[wb-cache] download-url answered ${res.status}`); + core_debug(`[wb-cache] restore-url answered ${res.status}`); return { kind: 'error' }; } catch (error) { - core_debug(`[wb-cache] download-url failed: ${error}`); + core_debug(`[wb-cache] restore-url failed: ${error}`); return { kind: 'error' }; } } +async function requestUpload(p, body) { + try { + const res = await fetch(backend_api_endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const b = (await res.json()); + return { kind: 'grant', url: b.url }; + } + if (res.status === 409) { + return { kind: 'locked' }; + } + if (res.status === 403) { + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] ${p} answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] ${p} failed: ${error}`); + return { kind: 'error' }; + } +} +// Cold repo: request the single-flight grant to build + upload the base mirror. +async function requestBaseUpload(repoKey) { + return requestUpload('base/upload-url', { repo_key: repoKey }); +} +// Warm repo: request the grant to overwrite this branch's delta bundle. +async function requestBranchUpload(repoKey, ref, sha) { + return requestUpload('branch/upload-url', { repo_key: repoKey, ref, sha }); +} ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ @@ -41743,23 +41786,29 @@ async function lookupSnapshot(repoKey, sha) { -// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch -// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are -// immutable (no expiry). Fail-open: any error degrades to stock behavior. +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; -// Logged at debug (normal state outside WarpBuild); other reasons log at info. +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Logged at info when off; the WARPBUILD_* env is only present on our runners. const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; -let decision = 'off'; -// Null = attempt the cache; else a reason to log. Only the default checkout shape -// is served. +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base'; +const BRANCH_REFNS = 'refs/wb/branch'; +const UPLOAD_TIP_REF = 'refs/wb/tip'; +let plan = { mode: 'off', repoKey: '', refKey: '' }; +// Null = attempt the cache; else a reason to log. The mirror serves full history for any +// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. function getMirrorCacheSkipReason(settings) { if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { return SKIP_NOT_WARPBUILD; } - const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; - if (!repoKey) { + if (!process.env['GITHUB_REPOSITORY_ID']) { return 'GITHUB_REPOSITORY_ID is not set'; } const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; @@ -41773,12 +41822,6 @@ function getMirrorCacheSkipReason(settings) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on'; } - if (settings.fetchDepth !== 1) { - return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`; - } - if (settings.fetchTags) { - return 'fetch-tags is enabled'; - } if (settings.filter) { return 'a fetch filter is configured'; } @@ -41786,51 +41829,49 @@ function getMirrorCacheSkipReason(settings) { return 'sparse checkout is configured'; } if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the snapshot)'; - } - if (settings.ref && computeDestinationRef(settings.ref) === null) { - return `ref '${settings.ref}' has no cacheable destination ref`; + return 'lfs is enabled (lfs objects are not in the mirror)'; } return null; } -// The local ref the fetch would have created ('' = none needed, null = uncacheable). -function computeDestinationRef(ref) { - if (!ref) { +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +function computeRefKey(settings) { + const baseRef = process.env['GITHUB_BASE_REF']; // set on pull_request events + if (baseRef) { + return baseRef; + } + const ref = process.env['GITHUB_REF'] || settings.ref || ''; + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length); + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { return ''; } - const upper = ref.toUpperCase(); - if (upper.startsWith('REFS/HEADS/')) { - return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`; - } - if (upper.startsWith('REFS/PULL/')) { - return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`; - } - if (upper.startsWith('REFS/TAGS/')) { - return ref; - } - return null; + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref; } -// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. -// Sets the `cache-hit` action output (true only on a restore). +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). async function setup(settings) { - const restored = await setupImpl(settings); - setOutput('cache-hit', restored ? 'true' : 'false'); - return restored; + const mode = await setupImpl(settings); + setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false'); + return mode; } async function setupImpl(settings) { - decision = 'off'; + plan = { mode: 'off', repoKey: '', refKey: '' }; const skipReason = getMirrorCacheSkipReason(settings); if (skipReason) { - info(`WarpBuild snapshot cache skipped: ${skipReason}`); - return false; + info(`WarpBuild mirror cache skipped: ${skipReason}`); + return 'off'; } - startGroup('WarpBuild: checkout snapshot cache'); + startGroup('WarpBuild: checkout mirror cache'); try { return await setupInner(settings); } catch (error) { - warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); - return false; + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + return 'off'; } finally { endGroup(); @@ -41838,140 +41879,196 @@ async function setupImpl(settings) { } async function setupInner(settings) { const repoKey = process.env['GITHUB_REPOSITORY_ID']; - const sha = settings.commit; - // The restore/upload paths shell out to `tar`; without it, fall back cleanly. - if (!(await which('tar', false))) { - info('tar not found on PATH; using standard checkout'); - return false; - } - const lookup = await lookupSnapshot(repoKey, sha); + const refKey = computeRefKey(settings); + plan = { mode: 'off', repoKey, refKey }; + const lookup = await lookupRestore(repoKey, refKey); if (lookup.kind === 'disabled') { - info('Snapshot cache is disabled by the backend for this organization'); - return false; + info('Mirror cache is disabled by the backend for this organization'); + return 'off'; } if (lookup.kind === 'error') { - info('Snapshot cache backend unavailable; using standard checkout'); - return false; + info('Mirror cache backend unavailable; using standard checkout'); + return 'off'; } - if (lookup.kind === 'miss') { - info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); - decision = 'miss'; - return false; + if (lookup.kind === 'cold') { + const grant = await requestBaseUpload(repoKey); + if (grant.kind === 'grant') { + info('Cold repo: building the base mirror this run (full fetch, then upload)'); + plan = { mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url }; + return 'cold-build'; + } + info('Cold repo: base build already in progress elsewhere; using standard checkout'); + return 'off'; } - info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); - if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { - return false; + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS); + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS); + } + catch (error) { + info(`Branch delta seed skipped (${error}); base only`); + } } - info('Snapshot restored; skipping the GitHub fetch entirely'); - return true; + info(`Seeded base mirror${lookup.branch ? ' + branch delta' : ''}; the GitHub fetch will be a tip delta`); + plan = { mode: 'seeded', repoKey, refKey }; + return 'seeded'; } -// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the -// snapshot after the job exits (keeping tar+upload off the customer's billed time). +// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) +// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. async function contribute(settings) { - if (decision !== 'miss') { - return; - } try { - await writeSnapshotManifest(settings); + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings); + } + else if (plan.mode === 'seeded') { + await uploadBranchDelta(settings); + } } catch (error) { - warning(`Snapshot manifest not written: ${error}`); + warning(`WarpBuild mirror upload skipped: ${error}`); } } -// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` -// component — the tar is remote and extracted into .git, so a crafted member could -// escape (e.g. hooks/, ../) and run code during checkout. -async function assertSafeTarMembers(tar) { - let listing = ''; - await exec_exec('tar', ['-tf', tar], { - silent: true, - listeners: { stdout: (d) => (listing += d.toString()) } - }); - for (const raw of listing.split('\n')) { - const member = raw.trim(); - if (!member) { - continue; - } - const top = member.replace(/^\.\//, '').split('/')[0]; - if (member.startsWith('/') || - member.split('/').includes('..') || - (top !== 'objects' && top !== 'shallow')) { - throw new Error(`unexpected snapshot tar member: ${member}`); - } - } -} -async function restoreSnapshot(settings, url, sha) { - const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); - const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`); +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle(repoPath, url, refNs) { + const tmp = tempBundlePath('seed'); try { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }); - if (!res.ok || !res.body) { - throw new Error(`snapshot download answered ${res.status}`); - } - await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); - await assertSafeTarMembers(tmpTar); - await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); - const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); - if (check !== 0) { - throw new Error(`restored snapshot does not contain ${sha}`); - } - // The ref the skipped fetch would have created; upstream's verification needs it. - const dstRef = computeDestinationRef(settings.ref); - if (dstRef) { - await exec_exec('git', [ - '-C', - settings.repositoryPath, - 'update-ref', - dstRef, - sha - ]); - } - return true; - } - catch (error) { - warning(`Snapshot restore failed: ${error}`); - await resetGitObjects(gitDir); - return false; + await downloadTo(url, tmp); + await exec_exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]); } finally { - await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); } } -// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` -// creates, so a failed restore leaves no trace for the fallback fetch. +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings) { + if (!plan.baseUploadUrl) { + return; + } + const tmp = tempBundlePath('base'); + try { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]); + await httpPut(plan.baseUploadUrl, tmp); + info('Uploaded base mirror'); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } +} +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings) { + if (!plan.refKey) { + return; // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath; + const sha = settings.commit; + const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha); + if (grant.kind !== 'grant') { + info(`Branch delta upload skipped (${grant.kind})`); + return; + } + const excludes = await baseRefExcludes(repoPath); + if (excludes.length === 0) { + info('No base refs to diff against; branch delta skipped'); + return; + } + const tmp = tempBundlePath('branch'); + try { + await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); + await exec_exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + ...excludes + ]); + await httpPut(grant.url, tmp); + info(`Uploaded branch delta for '${plan.refKey}'`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], { + ignoreReturnCode: true + }); + } +} +// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle +// base-relative (and therefore order-free). +async function baseRefExcludes(repoPath) { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out + .split('\n') + .map(s => s.trim()) + .filter(Boolean); +} +async function downloadTo(url, dest) { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }); + if (!res.ok || !res.body) { + throw new Error(`download answered ${res.status}`); + } + await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(dest)); +} +async function httpPut(url, file) { + const stat = await external_fs_namespaceObject.promises.stat(file); + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: external_fs_namespaceObject.createReadStream(file), + duplex: 'half', + headers: { 'content-length': String(stat.size) }, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + }); + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`); + } +} +function tempBundlePath(tag) { + return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`); +} +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. async function resetGitObjects(gitDir) { - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { + await fs.promises.rm(path.join(gitDir, 'objects'), { recursive: true, force: true }); - await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { + await fs.promises.rm(path.join(gitDir, 'shallow'), { force: true }); + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { recursive: true }); - await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { recursive: true }); } -// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which -// is /_work/_temp — host-visible to the agent (it knows the runner dir). -function manifestDir() { - const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir(); - return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots'); -} -// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. -async function writeSnapshotManifest(settings) { - const sha = settings.commit; - const manifest = { - repo_key: process.env['GITHUB_REPOSITORY_ID'], - sha, - git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git') - }; - const dir = manifestDir(); - await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true }); - await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest)); - info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`); +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +async function hasGit() { + return Boolean(await io.which('git', false)); } ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -42007,7 +42104,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; - let warpbuildRestored = false; + let warpbuildMode = 'off'; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -42059,7 +42156,7 @@ async function getSource(settings) { await git.remoteAdd('origin', repositoryUrl); endGroup(); // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped - warpbuildRestored = await setup(settings); + warpbuildMode = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -42099,10 +42196,17 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (warpbuildRestored) { - info('Skipping fetch: checkout was restored from the snapshot cache'); + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. + const refSpec = getRefSpec(settings.ref, settings.commit); + await git.fetch(refSpec, fetchOptions); + if (!(await testRef(git, settings.ref, settings.commit))) { + throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.`); + } } - else if (settings.fetchDepth <= 0) { + else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 3b5adbc..fbdf9d7 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -41,7 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null - let warpbuildRestored = false + let warpbuildMode: warpbuildMirror.MirrorMode = 'off' try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -134,7 +134,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped - warpbuildRestored = await warpbuildMirror.setup(settings) + warpbuildMode = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -190,9 +190,18 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (warpbuildRestored) { - core.info('Skipping fetch: checkout was restored from the snapshot cache') - } else if (settings.fetchDepth <= 0) { + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + throw new Error( + `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.` + ) + } + } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts index 0f8c379..b052abc 100644 --- a/src/warpbuild/backend-api.ts +++ b/src/warpbuild/backend-api.ts @@ -1,21 +1,30 @@ /* eslint-disable i18n-text/no-en -- upstream convention */ import * as core from '@actions/core' -// Client for backend-core's git-mirror download-url endpoint, authed by the runner -// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable -// org; else = fall back. The upload half runs in the warpbuild-agent after the job. +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. const API_TIMEOUT_MS = 10_000 -export interface SnapshotDownloadInfo { +export interface Presigned { url: string - size_bytes: number - created_at: string + size_bytes?: number } -export type SnapshotLookup = - | {kind: 'hit'; info: SnapshotDownloadInfo} - | {kind: 'miss'} +// GET restore-url: what to seed before the delta fetch. 'cold' = no base yet. +export type RestoreLookup = + | {kind: 'restore'; base: Presigned; branch: Presigned | null} + | {kind: 'cold'} + | {kind: 'disabled'} + | {kind: 'error'} + +// POST *upload-url: a presigned PUT, or a signal to skip (locked/disabled). +export type UploadGrant = + | {kind: 'grant'; url: string} + | {kind: 'locked'} | {kind: 'disabled'} | {kind: 'error'} @@ -27,34 +36,85 @@ function authHeader(): string { return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` } -export async function lookupSnapshot( +function endpoint(p: string): string { + return `${baseUrl()}/api/v1/git-mirrors/${p}` +} + +export async function lookupRestore( repoKey: string, - sha: string -): Promise { + ref: string +): Promise { try { const res = await fetch( - `${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( + `${endpoint('restore-url')}?repo_key=${encodeURIComponent( repoKey - )}&sha=${encodeURIComponent(sha)}`, + )}&ref=${encodeURIComponent(ref)}`, { headers: {authorization: authHeader()}, signal: AbortSignal.timeout(API_TIMEOUT_MS) } ) if (res.status === 200) { - return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} + const body = (await res.json()) as { + base: Presigned + branch: Presigned | null + } + return {kind: 'restore', base: body.base, branch: body.branch ?? null} } if (res.status === 404) { - return {kind: 'miss'} + return {kind: 'cold'} } if (res.status === 403) { - core.debug(`[wb-cache] download-url answered 403 (disabled)`) + core.debug('[wb-cache] restore-url answered 403 (disabled)') return {kind: 'disabled'} } - core.debug(`[wb-cache] download-url answered ${res.status}`) + core.debug(`[wb-cache] restore-url answered ${res.status}`) return {kind: 'error'} } catch (error) { - core.debug(`[wb-cache] download-url failed: ${error}`) + core.debug(`[wb-cache] restore-url failed: ${error}`) return {kind: 'error'} } } + +async function requestUpload(p: string, body: object): Promise { + try { + const res = await fetch(endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const b = (await res.json()) as {url: string} + return {kind: 'grant', url: b.url} + } + if (res.status === 409) { + return {kind: 'locked'} + } + if (res.status === 403) { + return {kind: 'disabled'} + } + core.debug(`[wb-cache] ${p} answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] ${p} failed: ${error}`) + return {kind: 'error'} + } +} + +// Cold repo: request the single-flight grant to build + upload the base mirror. +export async function requestBaseUpload(repoKey: string): Promise { + return requestUpload('base/upload-url', {repo_key: repoKey}) +} + +// Warm repo: request the grant to overwrite this branch's delta bundle. +export async function requestBranchUpload( + repoKey: string, + ref: string, + sha: string +): Promise { + return requestUpload('branch/upload-url', {repo_key: repoKey, ref, sha}) +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 6b54721..14ddefe 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -10,22 +10,38 @@ import {pipeline} from 'stream/promises' import {IGitSourceSettings} from '../git-source-settings.js' import * as api from './backend-api.js' -// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch -// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are -// immutable (no expiry). Fail-open: any error degrades to stock behavior. +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 +const UPLOAD_TIMEOUT_MS = 15 * 60_000 -// Logged at debug (normal state outside WarpBuild); other reasons log at info. +// Logged at info when off; the WARPBUILD_* env is only present on our runners. export const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)' const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ -let decision: 'off' | 'miss' = 'off' +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base' +const BRANCH_REFNS = 'refs/wb/branch' +const UPLOAD_TIP_REF = 'refs/wb/tip' -// Null = attempt the cache; else a reason to log. Only the default checkout shape -// is served. +export type MirrorMode = 'off' | 'seeded' | 'cold-build' + +interface Plan { + mode: MirrorMode + repoKey: string + refKey: string + baseUploadUrl?: string // cold-build only +} +let plan: Plan = {mode: 'off', repoKey: '', refKey: ''} + +// Null = attempt the cache; else a reason to log. The mirror serves full history for any +// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { @@ -35,8 +51,7 @@ export function getMirrorCacheSkipReason( ) { return SKIP_NOT_WARPBUILD } - const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' - if (!repoKey) { + if (!process.env['GITHUB_REPOSITORY_ID']) { return 'GITHUB_REPOSITORY_ID is not set' } const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` @@ -53,12 +68,6 @@ export function getMirrorCacheSkipReason( if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on' } - if (settings.fetchDepth !== 1) { - return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1` - } - if (settings.fetchTags) { - return 'fetch-tags is enabled' - } if (settings.filter) { return 'a fetch filter is configured' } @@ -66,191 +75,266 @@ export function getMirrorCacheSkipReason( return 'sparse checkout is configured' } if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the snapshot)' - } - if (settings.ref && computeDestinationRef(settings.ref) === null) { - return `ref '${settings.ref}' has no cacheable destination ref` + return 'lfs is enabled (lfs objects are not in the mirror)' } return null } -// The local ref the fetch would have created ('' = none needed, null = uncacheable). -export function computeDestinationRef(ref: string): string | null { - if (!ref) { +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +export function computeRefKey(settings: IGitSourceSettings): string { + const baseRef = process.env['GITHUB_BASE_REF'] // set on pull_request events + if (baseRef) { + return baseRef + } + const ref = process.env['GITHUB_REF'] || settings.ref || '' + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length) + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { return '' } - const upper = ref.toUpperCase() - if (upper.startsWith('REFS/HEADS/')) { - return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}` - } - if (upper.startsWith('REFS/PULL/')) { - return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}` - } - if (upper.startsWith('REFS/TAGS/')) { - return ref - } - return null + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref } -// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. -// Sets the `cache-hit` action output (true only on a restore). -export async function setup(settings: IGitSourceSettings): Promise { - const restored = await setupImpl(settings) - core.setOutput('cache-hit', restored ? 'true' : 'false') - return restored +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). +export async function setup(settings: IGitSourceSettings): Promise { + const mode = await setupImpl(settings) + core.setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false') + return mode } -async function setupImpl(settings: IGitSourceSettings): Promise { - decision = 'off' +async function setupImpl(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} const skipReason = getMirrorCacheSkipReason(settings) if (skipReason) { - core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) - return false + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + return 'off' } - core.startGroup('WarpBuild: checkout snapshot cache') + core.startGroup('WarpBuild: checkout mirror cache') try { return await setupInner(settings) } catch (error) { core.warning( - `WarpBuild snapshot cache unavailable, using standard checkout: ${error}` + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` ) - return false + return 'off' } finally { core.endGroup() } } -async function setupInner(settings: IGitSourceSettings): Promise { +async function setupInner(settings: IGitSourceSettings): Promise { const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string - const sha = settings.commit - - // The restore/upload paths shell out to `tar`; without it, fall back cleanly. - if (!(await io.which('tar', false))) { - core.info('tar not found on PATH; using standard checkout') - return false - } - - const lookup = await api.lookupSnapshot(repoKey, sha) + const refKey = computeRefKey(settings) + plan = {mode: 'off', repoKey, refKey} + const lookup = await api.lookupRestore(repoKey, refKey) if (lookup.kind === 'disabled') { - core.info('Snapshot cache is disabled by the backend for this organization') - return false + core.info('Mirror cache is disabled by the backend for this organization') + return 'off' } if (lookup.kind === 'error') { - core.info('Snapshot cache backend unavailable; using standard checkout') - return false + core.info('Mirror cache backend unavailable; using standard checkout') + return 'off' } - if (lookup.kind === 'miss') { + if (lookup.kind === 'cold') { + const grant = await api.requestBaseUpload(repoKey) + if (grant.kind === 'grant') { + core.info( + 'Cold repo: building the base mirror this run (full fetch, then upload)' + ) + plan = {mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url} + return 'cold-build' + } core.info( - `Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` + 'Cold repo: base build already in progress elsewhere; using standard checkout' ) - decision = 'miss' - return false + return 'off' } - core.info( - `Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` - ) - if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { - return false + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS) + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS) + } catch (error) { + core.info(`Branch delta seed skipped (${error}); base only`) + } } - core.info('Snapshot restored; skipping the GitHub fetch entirely') - return true + core.info( + `Seeded base mirror${ + lookup.branch ? ' + branch delta' : '' + }; the GitHub fetch will be a tip delta` + ) + plan = {mode: 'seeded', repoKey, refKey} + return 'seeded' } -// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the -// snapshot after the job exits (keeping tar+upload off the customer's billed time). +// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) +// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. export async function contribute(settings: IGitSourceSettings): Promise { - if (decision !== 'miss') { + try { + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings) + } else if (plan.mode === 'seeded') { + await uploadBranchDelta(settings) + } + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } +} + +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle( + repoPath: string, + url: string, + refNs: string +): Promise { + const tmp = tempBundlePath('seed') + try { + await downloadTo(url, tmp) + await exec.exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]) + } finally { + await fs.promises.rm(tmp, {force: true}) + } +} + +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings: IGitSourceSettings): Promise { + if (!plan.baseUploadUrl) { return } + const tmp = tempBundlePath('base') try { - await writeSnapshotManifest(settings) - } catch (error) { - core.warning(`Snapshot manifest not written: ${error}`) - } -} - -// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` -// component — the tar is remote and extracted into .git, so a crafted member could -// escape (e.g. hooks/, ../) and run code during checkout. -export async function assertSafeTarMembers(tar: string): Promise { - let listing = '' - await exec.exec('tar', ['-tf', tar], { - silent: true, - listeners: {stdout: (d: Buffer) => (listing += d.toString())} - }) - for (const raw of listing.split('\n')) { - const member = raw.trim() - if (!member) { - continue - } - const top = member.replace(/^\.\//, '').split('/')[0] - if ( - member.startsWith('/') || - member.split('/').includes('..') || - (top !== 'objects' && top !== 'shallow') - ) { - throw new Error(`unexpected snapshot tar member: ${member}`) - } - } -} - -async function restoreSnapshot( - settings: IGitSourceSettings, - url: string, - sha: string -): Promise { - const gitDir = path.join(settings.repositoryPath, '.git') - const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`) - try { - const res = await fetch(url, { - signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) - }) - if (!res.ok || !res.body) { - throw new Error(`snapshot download answered ${res.status}`) - } - await pipeline( - Readable.fromWeb(res.body as import('stream/web').ReadableStream), - fs.createWriteStream(tmpTar) - ) - await assertSafeTarMembers(tmpTar) - await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir]) - - const check = await exec.exec( - 'git', - ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], - {ignoreReturnCode: true} - ) - if (check !== 0) { - throw new Error(`restored snapshot does not contain ${sha}`) - } - - // The ref the skipped fetch would have created; upstream's verification needs it. - const dstRef = computeDestinationRef(settings.ref) - if (dstRef) { - await exec.exec('git', [ - '-C', - settings.repositoryPath, - 'update-ref', - dstRef, - sha - ]) - } - - return true - } catch (error) { - core.warning(`Snapshot restore failed: ${error}`) - await resetGitObjects(gitDir) - return false + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]) + await httpPut(plan.baseUploadUrl, tmp) + core.info('Uploaded base mirror') } finally { - await fs.promises.rm(tmpTar, {force: true}) + await fs.promises.rm(tmp, {force: true}) } } -// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` -// creates, so a failed restore leaves no trace for the fallback fetch. +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings: IGitSourceSettings): Promise { + if (!plan.refKey) { + return // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath + const sha = settings.commit + + const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha) + if (grant.kind !== 'grant') { + core.info(`Branch delta upload skipped (${grant.kind})`) + return + } + + const excludes = await baseRefExcludes(repoPath) + if (excludes.length === 0) { + core.info('No base refs to diff against; branch delta skipped') + return + } + + const tmp = tempBundlePath('branch') + try { + await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) + await exec.exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + ...excludes + ]) + await httpPut(grant.url, tmp) + core.info(`Uploaded branch delta for '${plan.refKey}'`) + } finally { + await fs.promises.rm(tmp, {force: true}) + await exec.exec( + 'git', + ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], + { + ignoreReturnCode: true + } + ) + } +} + +// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle +// base-relative (and therefore order-free). +async function baseRefExcludes(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + return out + .split('\n') + .map(s => s.trim()) + .filter(Boolean) +} + +async function downloadTo(url: string, dest: string): Promise { + const res = await fetch(url, { + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + }) + if (!res.ok || !res.body) { + throw new Error(`download answered ${res.status}`) + } + await pipeline( + Readable.fromWeb(res.body as import('stream/web').ReadableStream), + fs.createWriteStream(dest) + ) +} + +async function httpPut(url: string, file: string): Promise { + const stat = await fs.promises.stat(file) + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: fs.createReadStream(file) as any, + duplex: 'half', + headers: {'content-length': String(stat.size)}, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + } as any) + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`) + } +} + +function tempBundlePath(tag: string): string { + return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`) +} + +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. export async function resetGitObjects(gitDir: string): Promise { await fs.promises.rm(path.join(gitDir, 'objects'), { recursive: true, @@ -265,30 +349,7 @@ export async function resetGitObjects(gitDir: string): Promise { }) } -// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which -// is /_work/_temp — host-visible to the agent (it knows the runner dir). -export function manifestDir(): string { - const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir() - return path.join(runnerTemp, 'wb-snapshots') -} - -// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha. -async function writeSnapshotManifest( - settings: IGitSourceSettings -): Promise { - const sha = settings.commit - const manifest = { - repo_key: process.env['GITHUB_REPOSITORY_ID'] as string, - sha, - git_dir: path.join(settings.repositoryPath, '.git') - } - const dir = manifestDir() - await fs.promises.mkdir(dir, {recursive: true}) - await fs.promises.writeFile( - path.join(dir, `${sha}.json`), - JSON.stringify(manifest) - ) - core.info( - `Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job` - ) +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +export async function hasGit(): Promise { + return Boolean(await io.which('git', false)) }