get deltas from github

This commit is contained in:
darshanime 2026-07-13 08:48:11 +05:30
parent 41e5319596
commit 0c607adfe5
5 changed files with 706 additions and 540 deletions

View file

@ -6,26 +6,29 @@ import {
afterEach, afterEach,
afterAll afterAll
} from '@jest/globals' } from '@jest/globals'
import * as exec from '@actions/exec'
import * as fs from 'fs' import * as fs from 'fs'
import * as os from 'os' import * as os from 'os'
import * as path from 'path' import * as path from 'path'
import { import {
SKIP_NOT_WARPBUILD, SKIP_NOT_WARPBUILD,
assertSafeTarMembers, computeRefKey,
computeDestinationRef,
getMirrorCacheSkipReason, getMirrorCacheSkipReason,
manifestDir,
resetGitObjects resetGitObjects
} from '../src/warpbuild/mirror-cache.js' } from '../src/warpbuild/mirror-cache.js'
import {lookupSnapshot} from '../src/warpbuild/backend-api.js' import {
lookupRestore,
requestBaseUpload,
requestBranchUpload
} from '../src/warpbuild/backend-api.js'
import {IGitSourceSettings} from '../src/git-source-settings.js' import {IGitSourceSettings} from '../src/git-source-settings.js'
const WB_ENV = [ const WB_ENV = [
'WARPBUILD_RUNNER_VERIFICATION_TOKEN', 'WARPBUILD_RUNNER_VERIFICATION_TOKEN',
'WARPBUILD_HOST_URL', 'WARPBUILD_HOST_URL',
'GITHUB_REPOSITORY_ID', 'GITHUB_REPOSITORY_ID',
'GITHUB_REPOSITORY' 'GITHUB_REPOSITORY',
'GITHUB_REF',
'GITHUB_BASE_REF'
] ]
const savedEnv: {[key: string]: string | undefined} = {} const savedEnv: {[key: string]: string | undefined} = {}
for (const key of WB_ENV) { for (const key of WB_ENV) {
@ -57,9 +60,11 @@ function setWarpBuildEnv(): void {
process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev'
process.env['GITHUB_REPOSITORY_ID'] = '123456789' process.env['GITHUB_REPOSITORY_ID'] = '123456789'
process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world'
delete process.env['GITHUB_REF']
delete process.env['GITHUB_BASE_REF']
} }
describe('warpbuild snapshot cache', () => { describe('warpbuild mirror cache', () => {
beforeEach(() => { beforeEach(() => {
setWarpBuildEnv() setWarpBuildEnv()
}) })
@ -75,9 +80,7 @@ describe('warpbuild snapshot cache', () => {
}) })
describe('getMirrorCacheSkipReason', () => { describe('getMirrorCacheSkipReason', () => {
const winOnly = process.platform === 'win32' ? it.skip : it it('returns null for the default checkout shape', () => {
winOnly('returns null for the default checkout shape', () => {
expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() expect(getMirrorCacheSkipReason(settingsFor())).toBeNull()
}) })
@ -86,7 +89,7 @@ describe('warpbuild snapshot cache', () => {
expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD)
}) })
it('reports repository: inputs that are not the workflow repo', () => { it('reports repository inputs that are not the workflow repo', () => {
expect( expect(
getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'}))
).toBe( ).toBe(
@ -103,19 +106,15 @@ describe('warpbuild snapshot cache', () => {
) )
}) })
it('only serves fetch-depth 1', () => { it('serves any fetch-depth and fetch-tags (mirror is full history)', () => {
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBe( expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull()
'fetch-depth is 0, cache only serves fetch-depth 1' expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull()
) expect(
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBe( getMirrorCacheSkipReason(settingsFor({fetchTags: true}))
'fetch-depth is 50, cache only serves fetch-depth 1' ).toBeNull()
)
}) })
it('skips fetch-tags, filters, sparse and lfs checkouts', () => { it('skips filters, sparse and lfs checkouts', () => {
expect(getMirrorCacheSkipReason(settingsFor({fetchTags: true}))).toBe(
'fetch-tags is enabled'
)
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
'a fetch filter is configured' 'a fetch filter is configured'
) )
@ -123,18 +122,36 @@ describe('warpbuild snapshot cache', () => {
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
).toBe('sparse checkout is configured') ).toBe('sparse checkout is configured')
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe(
'lfs is enabled (lfs objects are not in the snapshot)' 'lfs is enabled (lfs objects are not in the mirror)'
) )
}) })
})
winOnly('accepts sha-only checkouts (empty ref)', () => { describe('computeRefKey', () => {
expect(getMirrorCacheSkipReason(settingsFor({ref: ''}))).toBeNull() it('keys on the pushed branch (GITHUB_REF)', () => {
process.env['GITHUB_REF'] = 'refs/heads/feature/x'
expect(computeRefKey(settingsFor())).toBe('feature/x')
}) })
it('skips unqualified refs', () => { it('keys on the base branch for pull requests (merge sha is synthetic)', () => {
expect(getMirrorCacheSkipReason(settingsFor({ref: 'main'}))).toBe( process.env['GITHUB_REF'] = 'refs/pull/42/merge'
"ref 'main' has no cacheable destination ref" process.env['GITHUB_BASE_REF'] = 'main'
) expect(computeRefKey(settingsFor())).toBe('main')
})
it('is empty for tags and bare pull refs (base only, no roll)', () => {
process.env['GITHUB_REF'] = 'refs/tags/v1.2.3'
expect(computeRefKey(settingsFor())).toBe('')
process.env['GITHUB_REF'] = 'refs/pull/42/merge'
expect(computeRefKey(settingsFor())).toBe('')
})
it('is empty for a detached sha checkout', () => {
expect(computeRefKey(settingsFor({ref: SHA}))).toBe('')
})
it('passes through an already-short branch name', () => {
expect(computeRefKey(settingsFor({ref: 'main'}))).toBe('main')
}) })
}) })
@ -144,7 +161,6 @@ describe('warpbuild snapshot cache', () => {
path.join(os.tmpdir(), 'wb-reset-') path.join(os.tmpdir(), 'wb-reset-')
) )
try { try {
// Simulate a partially-restored .git: stray objects, a pack, a shallow file.
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true recursive: true
}) })
@ -152,179 +168,95 @@ describe('warpbuild snapshot cache', () => {
path.join(gitDir, 'objects', 'ab'), path.join(gitDir, 'objects', 'ab'),
'junk loose object' 'junk loose object'
) )
await fs.promises.writeFile(
path.join(gitDir, 'objects', 'pack', 'pack-x.pack'),
'junk pack'
)
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n') await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
await resetGitObjects(gitDir) await resetGitObjects(gitDir)
// shallow gone; objects/ holds only the two empty init dirs.
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false) expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false) expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
expect( expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack']) ).toEqual(['info', 'pack'])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'pack'))
).toEqual([])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'info'))
).toEqual([])
} finally { } finally {
await fs.promises.rm(gitDir, {recursive: true, force: true}) await fs.promises.rm(gitDir, {recursive: true, force: true})
} }
}) })
it('is a no-op-safe on an already-clean .git', async () => {
const gitDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-reset-')
)
try {
await expect(resetGitObjects(gitDir)).resolves.toBeUndefined()
expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack'])
} finally {
await fs.promises.rm(gitDir, {recursive: true, force: true})
}
})
})
describe('assertSafeTarMembers', () => {
async function makeTar(
dir: string,
members: string[],
writeFiles = true
): Promise<string> {
if (writeFiles) {
for (const m of members) {
const abs = path.join(dir, m)
await fs.promises.mkdir(path.dirname(abs), {recursive: true})
await fs.promises.writeFile(abs, 'x')
}
}
const tar = path.join(dir, 'out.tar')
await exec.exec('tar', ['-cf', tar, '-C', dir, ...members])
return tar
}
it('accepts objects/ and shallow members', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow'])
await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined()
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
it('rejects members outside objects/ and shallow (e.g. hooks)', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit'])
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
/unexpected snapshot tar member/
)
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
it('rejects path traversal members', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
await fs.promises.mkdir(path.join(dir, 'sub'))
const tar = path.join(dir, 'evil.tar')
// -P preserves the ../ member instead of stripping it.
await fs.promises.writeFile(path.join(dir, 'evil'), 'x')
await exec.exec('tar', [
'-cPf',
tar,
'-C',
path.join(dir, 'sub'),
'../evil'
])
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
/unexpected snapshot tar member/
)
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
})
describe('computeDestinationRef', () => {
it('maps the refs the fetch would have created', () => {
expect(computeDestinationRef('refs/heads/main')).toBe(
'refs/remotes/origin/main'
)
expect(computeDestinationRef('refs/pull/42/merge')).toBe(
'refs/remotes/pull/42/merge'
)
expect(computeDestinationRef('refs/tags/v1.2.3')).toBe('refs/tags/v1.2.3')
expect(computeDestinationRef('')).toBe('')
expect(computeDestinationRef('main')).toBeNull()
})
}) })
describe('backend api http contract', () => { describe('backend api http contract', () => {
const realFetch = globalThis.fetch const realFetch = globalThis.fetch
let lastUrl = '' let lastUrl = ''
let lastInit: RequestInit | undefined
afterEach(() => { afterEach(() => {
globalThis.fetch = realFetch globalThis.fetch = realFetch
}) })
function stubFetch(status: number, body: unknown): void { function stubFetch(status: number, body: unknown): void {
globalThis.fetch = (async (input: unknown) => { globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
lastUrl = String(input) lastUrl = String(input)
lastInit = init
return new Response(JSON.stringify(body), {status}) return new Response(JSON.stringify(body), {status})
}) as typeof fetch }) as typeof fetch
} }
it('maps 200 to hit and keys by sha', async () => { it('lookupRestore maps 200 to a restore plan keyed by ref', async () => {
stubFetch(200, {url: 'https://s3/x', size_bytes: 42, created_at: 'now'}) stubFetch(200, {
const result = await lookupSnapshot('123', SHA) base: {url: 'https://s3/base', size_bytes: 42},
expect(result.kind).toBe('hit') branch: {url: 'https://s3/branch'}
expect(lastUrl).toContain(`sha=${SHA}`) })
const result = await lookupRestore('123', 'main')
expect(result.kind).toBe('restore')
if (result.kind === 'restore') {
expect(result.base.url).toBe('https://s3/base')
expect(result.branch?.url).toBe('https://s3/branch')
}
expect(lastUrl).toContain('ref=main')
}) })
it('maps 404 to miss (agent uploads after the job)', async () => { it('lookupRestore maps a null branch (base but no delta yet)', async () => {
stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) stubFetch(200, {base: {url: 'https://s3/base'}, branch: null})
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') const result = await lookupRestore('123', 'main')
}) expect(result.kind).toBe('restore')
if (result.kind === 'restore') {
it('maps 403 to disabled (backend kill switch)', async () => { expect(result.branch).toBeNull()
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
})
it('maps other statuses and network failures to error', async () => {
stubFetch(500, {})
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
globalThis.fetch = (async () => {
throw new Error('boom')
}) as typeof fetch
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
})
})
describe('manifestDir', () => {
const saved = process.env['RUNNER_TEMP']
afterEach(() => {
if (saved === undefined) {
delete process.env['RUNNER_TEMP']
} else {
process.env['RUNNER_TEMP'] = saved
} }
}) })
it('places manifests under RUNNER_TEMP/wb-snapshots', () => { it('lookupRestore maps 404 to cold, 403 to disabled, 500 to error', async () => {
process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x') stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
expect(manifestDir()).toBe( expect((await lookupRestore('123', 'main')).kind).toBe('cold')
path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots') stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupRestore('123', 'main')).kind).toBe('disabled')
stubFetch(500, {})
expect((await lookupRestore('123', 'main')).kind).toBe('error')
})
it('lookupRestore maps network failure to error', async () => {
globalThis.fetch = (async () => {
throw new Error('boom')
}) as typeof fetch
expect((await lookupRestore('123', 'main')).kind).toBe('error')
})
it('requestBaseUpload maps 200 to a grant, 409 to locked', async () => {
stubFetch(200, {url: 'https://s3/put-base'})
const grant = await requestBaseUpload('123')
expect(grant.kind).toBe('grant')
if (grant.kind === 'grant') {
expect(grant.url).toBe('https://s3/put-base')
}
expect(lastInit?.method).toBe('POST')
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
expect((await requestBaseUpload('123')).kind).toBe('locked')
})
it('requestBranchUpload maps 200 to a grant, 403 to disabled', async () => {
stubFetch(200, {url: 'https://s3/put-branch'})
expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe('grant')
stubFetch(403, {})
expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe(
'disabled'
) )
}) })
}) })

436
dist/index.js vendored
View file

@ -41697,9 +41697,11 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
/* eslint-disable i18n-text/no-en -- upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
// Client for backend-core's git-mirror download-url endpoint, authed by the runner // Client for backend-core's git-mirror endpoints, authed by the runner verification
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable // token. The action seeds from a bare base mirror + a per-branch delta bundle (both
// org; else = fall back. The upload half runs in the warpbuild-agent after the job. // presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed
// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call
// fails closed to stock checkout.
const API_TIMEOUT_MS = 10_000; const API_TIMEOUT_MS = 10_000;
function backend_api_baseUrl() { function backend_api_baseUrl() {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@ -41707,30 +41709,71 @@ function backend_api_baseUrl() {
function authHeader() { function authHeader() {
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`;
} }
async function lookupSnapshot(repoKey, sha) { function backend_api_endpoint(p) {
return `${backend_api_baseUrl()}/api/v1/git-mirrors/${p}`;
}
async function lookupRestore(repoKey, ref) {
try { try {
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent(repoKey)}&sha=${encodeURIComponent(sha)}`, { const res = await fetch(`${backend_api_endpoint('restore-url')}?repo_key=${encodeURIComponent(repoKey)}&ref=${encodeURIComponent(ref)}`, {
headers: { authorization: authHeader() }, headers: { authorization: authHeader() },
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
}); });
if (res.status === 200) { if (res.status === 200) {
return { kind: 'hit', info: (await res.json()) }; const body = (await res.json());
return { kind: 'restore', base: body.base, branch: body.branch ?? null };
} }
if (res.status === 404) { if (res.status === 404) {
return { kind: 'miss' }; return { kind: 'cold' };
} }
if (res.status === 403) { if (res.status === 403) {
core_debug(`[wb-cache] download-url answered 403 (disabled)`); core_debug('[wb-cache] restore-url answered 403 (disabled)');
return { kind: 'disabled' }; return { kind: 'disabled' };
} }
core_debug(`[wb-cache] download-url answered ${res.status}`); core_debug(`[wb-cache] restore-url answered ${res.status}`);
return { kind: 'error' }; return { kind: 'error' };
} }
catch (error) { catch (error) {
core_debug(`[wb-cache] download-url failed: ${error}`); core_debug(`[wb-cache] restore-url failed: ${error}`);
return { kind: 'error' }; return { kind: 'error' };
} }
} }
async function requestUpload(p, body) {
try {
const res = await fetch(backend_api_endpoint(p), {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
});
if (res.status === 200) {
const b = (await res.json());
return { kind: 'grant', url: b.url };
}
if (res.status === 409) {
return { kind: 'locked' };
}
if (res.status === 403) {
return { kind: 'disabled' };
}
core_debug(`[wb-cache] ${p} answered ${res.status}`);
return { kind: 'error' };
}
catch (error) {
core_debug(`[wb-cache] ${p} failed: ${error}`);
return { kind: 'error' };
}
}
// Cold repo: request the single-flight grant to build + upload the base mirror.
async function requestBaseUpload(repoKey) {
return requestUpload('base/upload-url', { repo_key: repoKey });
}
// Warm repo: request the grant to overwrite this branch's delta bundle.
async function requestBranchUpload(repoKey, ref, sha) {
return requestUpload('branch/upload-url', { repo_key: repoKey, ref, sha });
}
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
@ -41743,23 +41786,29 @@ async function lookupSnapshot(repoKey, sha) {
// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch // WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are // S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
// immutable (no expiry). Fail-open: any error degrades to stock behavior. // only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
// so order-free); a cold repo single-flights the base build. Fail-open: any error →
// standard checkout.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
// Logged at debug (normal state outside WarpBuild); other reasons log at info. const UPLOAD_TIMEOUT_MS = 15 * 60_000;
// Logged at info when off; the WARPBUILD_* env is only present on our runners.
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
let decision = 'off'; // Seeded bundles land here, out of the user's ref space; excluded objects for deltas.
// Null = attempt the cache; else a reason to log. Only the default checkout shape const BASE_REFNS = 'refs/wb/base';
// is served. const BRANCH_REFNS = 'refs/wb/branch';
const UPLOAD_TIP_REF = 'refs/wb/tip';
let plan = { mode: 'off', repoKey: '', refKey: '' };
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
function getMirrorCacheSkipReason(settings) { function getMirrorCacheSkipReason(settings) {
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
!process.env['WARPBUILD_HOST_URL']) { !process.env['WARPBUILD_HOST_URL']) {
return SKIP_NOT_WARPBUILD; return SKIP_NOT_WARPBUILD;
} }
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''; if (!process.env['GITHUB_REPOSITORY_ID']) {
if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set'; return 'GITHUB_REPOSITORY_ID is not set';
} }
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`;
@ -41773,12 +41822,6 @@ function getMirrorCacheSkipReason(settings) {
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on'; return 'no exact commit sha to key on';
} }
if (settings.fetchDepth !== 1) {
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`;
}
if (settings.fetchTags) {
return 'fetch-tags is enabled';
}
if (settings.filter) { if (settings.filter) {
return 'a fetch filter is configured'; return 'a fetch filter is configured';
} }
@ -41786,51 +41829,49 @@ function getMirrorCacheSkipReason(settings) {
return 'sparse checkout is configured'; return 'sparse checkout is configured';
} }
if (settings.lfs) { if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the snapshot)'; return 'lfs is enabled (lfs objects are not in the mirror)';
}
if (settings.ref && computeDestinationRef(settings.ref) === null) {
return `ref '${settings.ref}' has no cacheable destination ref`;
} }
return null; return null;
} }
// The local ref the fetch would have created ('' = none needed, null = uncacheable). // The durable branch to key the per-branch bundle on. For pull_request events the merge
function computeDestinationRef(ref) { // SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for
if (!ref) { // tags / detached HEAD → seed the base only, upload nothing.
function computeRefKey(settings) {
const baseRef = process.env['GITHUB_BASE_REF']; // set on pull_request events
if (baseRef) {
return baseRef;
}
const ref = process.env['GITHUB_REF'] || settings.ref || '';
if (ref.startsWith('refs/heads/')) {
return ref.substring('refs/heads/'.length);
}
if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) {
return ''; return '';
} }
const upper = ref.toUpperCase(); // Already a short branch name, or empty.
if (upper.startsWith('REFS/HEADS/')) { return SHA_PATTERN.test(ref) ? '' : ref;
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`;
}
if (upper.startsWith('REFS/PULL/')) {
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`;
}
if (upper.startsWith('REFS/TAGS/')) {
return ref;
}
return null;
} }
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. // Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps
// Sets the `cache-hit` action output (true only on a restore). // branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache).
async function setup(settings) { async function setup(settings) {
const restored = await setupImpl(settings); const mode = await setupImpl(settings);
setOutput('cache-hit', restored ? 'true' : 'false'); setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false');
return restored; return mode;
} }
async function setupImpl(settings) { async function setupImpl(settings) {
decision = 'off'; plan = { mode: 'off', repoKey: '', refKey: '' };
const skipReason = getMirrorCacheSkipReason(settings); const skipReason = getMirrorCacheSkipReason(settings);
if (skipReason) { if (skipReason) {
info(`WarpBuild snapshot cache skipped: ${skipReason}`); info(`WarpBuild mirror cache skipped: ${skipReason}`);
return false; return 'off';
} }
startGroup('WarpBuild: checkout snapshot cache'); startGroup('WarpBuild: checkout mirror cache');
try { try {
return await setupInner(settings); return await setupInner(settings);
} }
catch (error) { catch (error) {
warning(`WarpBuild snapshot cache unavailable, using standard checkout: ${error}`); warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`);
return false; return 'off';
} }
finally { finally {
endGroup(); endGroup();
@ -41838,140 +41879,196 @@ async function setupImpl(settings) {
} }
async function setupInner(settings) { async function setupInner(settings) {
const repoKey = process.env['GITHUB_REPOSITORY_ID']; const repoKey = process.env['GITHUB_REPOSITORY_ID'];
const sha = settings.commit; const refKey = computeRefKey(settings);
// The restore/upload paths shell out to `tar`; without it, fall back cleanly. plan = { mode: 'off', repoKey, refKey };
if (!(await which('tar', false))) { const lookup = await lookupRestore(repoKey, refKey);
info('tar not found on PATH; using standard checkout');
return false;
}
const lookup = await lookupSnapshot(repoKey, sha);
if (lookup.kind === 'disabled') { if (lookup.kind === 'disabled') {
info('Snapshot cache is disabled by the backend for this organization'); info('Mirror cache is disabled by the backend for this organization');
return false; return 'off';
} }
if (lookup.kind === 'error') { if (lookup.kind === 'error') {
info('Snapshot cache backend unavailable; using standard checkout'); info('Mirror cache backend unavailable; using standard checkout');
return false; return 'off';
} }
if (lookup.kind === 'miss') { if (lookup.kind === 'cold') {
info(`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded`); const grant = await requestBaseUpload(repoKey);
decision = 'miss'; if (grant.kind === 'grant') {
return false; info('Cold repo: building the base mirror this run (full fetch, then upload)');
plan = { mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url };
return 'cold-build';
}
info('Cold repo: base build already in progress elsewhere; using standard checkout');
return 'off';
} }
info(`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)`); // Restore: seed the base, then this branch's delta if present.
if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS);
return false; if (lookup.branch) {
try {
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS);
}
catch (error) {
info(`Branch delta seed skipped (${error}); base only`);
}
} }
info('Snapshot restored; skipping the GitHub fetch entirely'); info(`Seeded base mirror${lookup.branch ? ' + branch delta' : ''}; the GitHub fetch will be a tip delta`);
return true; plan = { mode: 'seeded', repoKey, refKey };
return 'seeded';
} }
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the // Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
// snapshot after the job exits (keeping tar+upload off the customer's billed time). // or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
async function contribute(settings) { async function contribute(settings) {
if (decision !== 'miss') {
return;
}
try { try {
await writeSnapshotManifest(settings); if (plan.mode === 'cold-build') {
await uploadBaseMirror(settings);
}
else if (plan.mode === 'seeded') {
await uploadBranchDelta(settings);
}
} }
catch (error) { catch (error) {
warning(`Snapshot manifest not written: ${error}`); warning(`WarpBuild mirror upload skipped: ${error}`);
} }
} }
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` // Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
// component — the tar is remote and extracted into .git, so a crafted member could // (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
// escape (e.g. hooks/, ../) and run code during checkout. // nothing is extracted into .git directly.
async function assertSafeTarMembers(tar) { async function seedBundle(repoPath, url, refNs) {
let listing = ''; const tmp = tempBundlePath('seed');
await exec_exec('tar', ['-tf', tar], {
silent: true,
listeners: { stdout: (d) => (listing += d.toString()) }
});
for (const raw of listing.split('\n')) {
const member = raw.trim();
if (!member) {
continue;
}
const top = member.replace(/^\.\//, '').split('/')[0];
if (member.startsWith('/') ||
member.split('/').includes('..') ||
(top !== 'objects' && top !== 'shallow')) {
throw new Error(`unexpected snapshot tar member: ${member}`);
}
}
}
async function restoreSnapshot(settings, url, sha) {
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
try { try {
const res = await fetch(url, { await downloadTo(url, tmp);
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) await exec_exec('git', [
}); '-C',
if (!res.ok || !res.body) { repoPath,
throw new Error(`snapshot download answered ${res.status}`); 'fetch',
} '--quiet',
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar)); '--no-tags',
await assertSafeTarMembers(tmpTar); tmp,
await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]); `+refs/*:${refNs}/*`
const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true }); ]);
if (check !== 0) {
throw new Error(`restored snapshot does not contain ${sha}`);
}
// The ref the skipped fetch would have created; upstream's verification needs it.
const dstRef = computeDestinationRef(settings.ref);
if (dstRef) {
await exec_exec('git', [
'-C',
settings.repositoryPath,
'update-ref',
dstRef,
sha
]);
}
return true;
}
catch (error) {
warning(`Snapshot restore failed: ${error}`);
await resetGitObjects(gitDir);
return false;
} }
finally { finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); await external_fs_namespaceObject.promises.rm(tmp, { force: true });
} }
} }
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` // Cold-build: after the forced full fetch the repo holds all branches under
// creates, so a failed restore leaves no trace for the fallback fetch. // refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
async function uploadBaseMirror(settings) {
if (!plan.baseUploadUrl) {
return;
}
const tmp = tempBundlePath('base');
try {
await exec_exec('git', [
'-C',
settings.repositoryPath,
'bundle',
'create',
tmp,
'--remotes=origin',
'--tags'
]);
await httpPut(plan.baseUploadUrl, tmp);
info('Uploaded base mirror');
}
finally {
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
}
}
// Seeded: build the base-relative delta (target sha, excluding everything in the base)
// and overwrite this branch's bundle. Guarded by a per-branch lock server-side.
async function uploadBranchDelta(settings) {
if (!plan.refKey) {
return; // tag / detached HEAD: nothing to roll
}
const repoPath = settings.repositoryPath;
const sha = settings.commit;
const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha);
if (grant.kind !== 'grant') {
info(`Branch delta upload skipped (${grant.kind})`);
return;
}
const excludes = await baseRefExcludes(repoPath);
if (excludes.length === 0) {
info('No base refs to diff against; branch delta skipped');
return;
}
const tmp = tempBundlePath('branch');
try {
await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]);
await exec_exec('git', [
'-C',
repoPath,
'bundle',
'create',
tmp,
UPLOAD_TIP_REF,
...excludes
]);
await httpPut(grant.url, tmp);
info(`Uploaded branch delta for '${plan.refKey}'`);
}
finally {
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], {
ignoreReturnCode: true
});
}
}
// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle
// base-relative (and therefore order-free).
async function baseRefExcludes(repoPath) {
let out = '';
await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
return out
.split('\n')
.map(s => s.trim())
.filter(Boolean);
}
async function downloadTo(url, dest) {
const res = await fetch(url, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
});
if (!res.ok || !res.body) {
throw new Error(`download answered ${res.status}`);
}
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(dest));
}
async function httpPut(url, file) {
const stat = await external_fs_namespaceObject.promises.stat(file);
const res = await fetch(url, {
method: 'PUT',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body
body: external_fs_namespaceObject.createReadStream(file),
duplex: 'half',
headers: { 'content-length': String(stat.size) },
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed
});
if (!res.ok) {
throw new Error(`PUT answered ${res.status}`);
}
}
function tempBundlePath(tag) {
return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`);
}
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
async function resetGitObjects(gitDir) { async function resetGitObjects(gitDir) {
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), { await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true, recursive: true,
force: true force: true
}); });
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true }); await fs.promises.rm(path.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), { await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
recursive: true recursive: true
}); });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), { await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true recursive: true
}); });
} }
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which // Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`.
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir). async function hasGit() {
function manifestDir() { return Boolean(await io.which('git', false));
const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir();
return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots');
}
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(settings) {
const sha = settings.commit;
const manifest = {
repo_key: process.env['GITHUB_REPOSITORY_ID'],
sha,
git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git')
};
const dir = manifestDir();
await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true });
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest));
info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`);
} }
;// CONCATENATED MODULE: ./src/git-source-provider.ts ;// CONCATENATED MODULE: ./src/git-source-provider.ts
@ -42007,7 +42104,7 @@ async function getSource(settings) {
const git = await getGitCommandManager(settings); const git = await getGitCommandManager(settings);
endGroup(); endGroup();
let authHelper = null; let authHelper = null;
let warpbuildRestored = false; let warpbuildMode = 'off';
try { try {
if (git) { if (git) {
authHelper = createAuthHelper(git, settings); authHelper = createAuthHelper(git, settings);
@ -42059,7 +42156,7 @@ async function getSource(settings) {
await git.remoteAdd('origin', repositoryUrl); await git.remoteAdd('origin', repositoryUrl);
endGroup(); endGroup();
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
warpbuildRestored = await setup(settings); warpbuildMode = await setup(settings);
} }
// Disable automatic garbage collection // Disable automatic garbage collection
startGroup('Disabling automatic garbage collection'); startGroup('Disabling automatic garbage collection');
@ -42099,10 +42196,17 @@ async function getSource(settings) {
else if (settings.sparseCheckout) { else if (settings.sparseCheckout) {
fetchOptions.filter = 'blob:none'; fetchOptions.filter = 'blob:none';
} }
if (warpbuildRestored) { if (warpbuildMode === 'seeded') {
info('Skipping fetch: checkout was restored from the snapshot cache'); // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
// objects. No depth — the base is full history, so this stays non-shallow.
const refSpec = getRefSpec(settings.ref, settings.commit);
await git.fetch(refSpec, fetchOptions);
if (!(await testRef(git, settings.ref, settings.commit))) {
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`);
}
} }
else if (settings.fetchDepth <= 0) { else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
// Fetch all branches and tags // Fetch all branches and tags
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
await git.fetch(refSpec, fetchOptions); await git.fetch(refSpec, fetchOptions);

View file

@ -41,7 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
core.endGroup() core.endGroup()
let authHelper: gitAuthHelper.IGitAuthHelper | null = null let authHelper: gitAuthHelper.IGitAuthHelper | null = null
let warpbuildRestored = false let warpbuildMode: warpbuildMirror.MirrorMode = 'off'
try { try {
if (git) { if (git) {
authHelper = gitAuthHelper.createAuthHelper(git, settings) authHelper = gitAuthHelper.createAuthHelper(git, settings)
@ -134,7 +134,7 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
core.endGroup() core.endGroup()
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
warpbuildRestored = await warpbuildMirror.setup(settings) warpbuildMode = await warpbuildMirror.setup(settings)
} }
// Disable automatic garbage collection // Disable automatic garbage collection
@ -190,9 +190,18 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
fetchOptions.filter = 'blob:none' fetchOptions.filter = 'blob:none'
} }
if (warpbuildRestored) { if (warpbuildMode === 'seeded') {
core.info('Skipping fetch: checkout was restored from the snapshot cache') // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
} else if (settings.fetchDepth <= 0) { // objects. No depth — the base is full history, so this stays non-shallow.
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
await git.fetch(refSpec, fetchOptions)
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
throw new Error(
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`
)
}
} else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
// Fetch all branches and tags // Fetch all branches and tags
let refSpec = refHelper.getRefSpecForAllHistory( let refSpec = refHelper.getRefSpecForAllHistory(
settings.ref, settings.ref,

View file

@ -1,21 +1,30 @@
/* eslint-disable i18n-text/no-en -- upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
import * as core from '@actions/core' import * as core from '@actions/core'
// Client for backend-core's git-mirror download-url endpoint, authed by the runner // Client for backend-core's git-mirror endpoints, authed by the runner verification
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable // token. The action seeds from a bare base mirror + a per-branch delta bundle (both
// org; else = fall back. The upload half runs in the warpbuild-agent after the job. // presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed
// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call
// fails closed to stock checkout.
const API_TIMEOUT_MS = 10_000 const API_TIMEOUT_MS = 10_000
export interface SnapshotDownloadInfo { export interface Presigned {
url: string url: string
size_bytes: number size_bytes?: number
created_at: string
} }
export type SnapshotLookup = // GET restore-url: what to seed before the delta fetch. 'cold' = no base yet.
| {kind: 'hit'; info: SnapshotDownloadInfo} export type RestoreLookup =
| {kind: 'miss'} | {kind: 'restore'; base: Presigned; branch: Presigned | null}
| {kind: 'cold'}
| {kind: 'disabled'}
| {kind: 'error'}
// POST *upload-url: a presigned PUT, or a signal to skip (locked/disabled).
export type UploadGrant =
| {kind: 'grant'; url: string}
| {kind: 'locked'}
| {kind: 'disabled'} | {kind: 'disabled'}
| {kind: 'error'} | {kind: 'error'}
@ -27,34 +36,85 @@ function authHeader(): string {
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`
} }
export async function lookupSnapshot( function endpoint(p: string): string {
return `${baseUrl()}/api/v1/git-mirrors/${p}`
}
export async function lookupRestore(
repoKey: string, repoKey: string,
sha: string ref: string
): Promise<SnapshotLookup> { ): Promise<RestoreLookup> {
try { try {
const res = await fetch( const res = await fetch(
`${baseUrl()}/api/v1/git-mirrors/download-url?repo_key=${encodeURIComponent( `${endpoint('restore-url')}?repo_key=${encodeURIComponent(
repoKey repoKey
)}&sha=${encodeURIComponent(sha)}`, )}&ref=${encodeURIComponent(ref)}`,
{ {
headers: {authorization: authHeader()}, headers: {authorization: authHeader()},
signal: AbortSignal.timeout(API_TIMEOUT_MS) signal: AbortSignal.timeout(API_TIMEOUT_MS)
} }
) )
if (res.status === 200) { if (res.status === 200) {
return {kind: 'hit', info: (await res.json()) as SnapshotDownloadInfo} const body = (await res.json()) as {
base: Presigned
branch: Presigned | null
}
return {kind: 'restore', base: body.base, branch: body.branch ?? null}
} }
if (res.status === 404) { if (res.status === 404) {
return {kind: 'miss'} return {kind: 'cold'}
} }
if (res.status === 403) { if (res.status === 403) {
core.debug(`[wb-cache] download-url answered 403 (disabled)`) core.debug('[wb-cache] restore-url answered 403 (disabled)')
return {kind: 'disabled'} return {kind: 'disabled'}
} }
core.debug(`[wb-cache] download-url answered ${res.status}`) core.debug(`[wb-cache] restore-url answered ${res.status}`)
return {kind: 'error'} return {kind: 'error'}
} catch (error) { } catch (error) {
core.debug(`[wb-cache] download-url failed: ${error}`) core.debug(`[wb-cache] restore-url failed: ${error}`)
return {kind: 'error'} return {kind: 'error'}
} }
} }
async function requestUpload(p: string, body: object): Promise<UploadGrant> {
try {
const res = await fetch(endpoint(p), {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
})
if (res.status === 200) {
const b = (await res.json()) as {url: string}
return {kind: 'grant', url: b.url}
}
if (res.status === 409) {
return {kind: 'locked'}
}
if (res.status === 403) {
return {kind: 'disabled'}
}
core.debug(`[wb-cache] ${p} answered ${res.status}`)
return {kind: 'error'}
} catch (error) {
core.debug(`[wb-cache] ${p} failed: ${error}`)
return {kind: 'error'}
}
}
// Cold repo: request the single-flight grant to build + upload the base mirror.
export async function requestBaseUpload(repoKey: string): Promise<UploadGrant> {
return requestUpload('base/upload-url', {repo_key: repoKey})
}
// Warm repo: request the grant to overwrite this branch's delta bundle.
export async function requestBranchUpload(
repoKey: string,
ref: string,
sha: string
): Promise<UploadGrant> {
return requestUpload('branch/upload-url', {repo_key: repoKey, ref, sha})
}

View file

@ -10,22 +10,38 @@ import {pipeline} from 'stream/promises'
import {IGitSourceSettings} from '../git-source-settings.js' import {IGitSourceSettings} from '../git-source-settings.js'
import * as api from './backend-api.js' import * as api from './backend-api.js'
// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch // WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are // S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
// immutable (no expiry). Fail-open: any error degrades to stock behavior. // only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
// so order-free); a cold repo single-flights the base build. Fail-open: any error →
// standard checkout.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
const UPLOAD_TIMEOUT_MS = 15 * 60_000
// Logged at debug (normal state outside WarpBuild); other reasons log at info. // Logged at info when off; the WARPBUILD_* env is only present on our runners.
export const SKIP_NOT_WARPBUILD = export const SKIP_NOT_WARPBUILD =
'not running on a WarpBuild runner (WARPBUILD_* env not present)' 'not running on a WarpBuild runner (WARPBUILD_* env not present)'
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/
let decision: 'off' | 'miss' = 'off' // Seeded bundles land here, out of the user's ref space; excluded objects for deltas.
const BASE_REFNS = 'refs/wb/base'
const BRANCH_REFNS = 'refs/wb/branch'
const UPLOAD_TIP_REF = 'refs/wb/tip'
// Null = attempt the cache; else a reason to log. Only the default checkout shape export type MirrorMode = 'off' | 'seeded' | 'cold-build'
// is served.
interface Plan {
mode: MirrorMode
repoKey: string
refKey: string
baseUploadUrl?: string // cold-build only
}
let plan: Plan = {mode: 'off', repoKey: '', refKey: ''}
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
export function getMirrorCacheSkipReason( export function getMirrorCacheSkipReason(
settings: IGitSourceSettings settings: IGitSourceSettings
): string | null { ): string | null {
@ -35,8 +51,7 @@ export function getMirrorCacheSkipReason(
) { ) {
return SKIP_NOT_WARPBUILD return SKIP_NOT_WARPBUILD
} }
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '' if (!process.env['GITHUB_REPOSITORY_ID']) {
if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set' return 'GITHUB_REPOSITORY_ID is not set'
} }
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`
@ -53,12 +68,6 @@ export function getMirrorCacheSkipReason(
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on' return 'no exact commit sha to key on'
} }
if (settings.fetchDepth !== 1) {
return `fetch-depth is ${settings.fetchDepth}, cache only serves fetch-depth 1`
}
if (settings.fetchTags) {
return 'fetch-tags is enabled'
}
if (settings.filter) { if (settings.filter) {
return 'a fetch filter is configured' return 'a fetch filter is configured'
} }
@ -66,191 +75,266 @@ export function getMirrorCacheSkipReason(
return 'sparse checkout is configured' return 'sparse checkout is configured'
} }
if (settings.lfs) { if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the snapshot)' return 'lfs is enabled (lfs objects are not in the mirror)'
}
if (settings.ref && computeDestinationRef(settings.ref) === null) {
return `ref '${settings.ref}' has no cacheable destination ref`
} }
return null return null
} }
// The local ref the fetch would have created ('' = none needed, null = uncacheable). // The durable branch to key the per-branch bundle on. For pull_request events the merge
export function computeDestinationRef(ref: string): string | null { // SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for
if (!ref) { // tags / detached HEAD → seed the base only, upload nothing.
export function computeRefKey(settings: IGitSourceSettings): string {
const baseRef = process.env['GITHUB_BASE_REF'] // set on pull_request events
if (baseRef) {
return baseRef
}
const ref = process.env['GITHUB_REF'] || settings.ref || ''
if (ref.startsWith('refs/heads/')) {
return ref.substring('refs/heads/'.length)
}
if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) {
return '' return ''
} }
const upper = ref.toUpperCase() // Already a short branch name, or empty.
if (upper.startsWith('REFS/HEADS/')) { return SHA_PATTERN.test(ref) ? '' : ref
return `refs/remotes/origin/${ref.substring('refs/heads/'.length)}`
}
if (upper.startsWith('REFS/PULL/')) {
return `refs/remotes/pull/${ref.substring('refs/pull/'.length)}`
}
if (upper.startsWith('REFS/TAGS/')) {
return ref
}
return null
} }
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. // Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps
// Sets the `cache-hit` action output (true only on a restore). // branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache).
export async function setup(settings: IGitSourceSettings): Promise<boolean> { export async function setup(settings: IGitSourceSettings): Promise<MirrorMode> {
const restored = await setupImpl(settings) const mode = await setupImpl(settings)
core.setOutput('cache-hit', restored ? 'true' : 'false') core.setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false')
return restored return mode
} }
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> { async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
decision = 'off' plan = {mode: 'off', repoKey: '', refKey: ''}
const skipReason = getMirrorCacheSkipReason(settings) const skipReason = getMirrorCacheSkipReason(settings)
if (skipReason) { if (skipReason) {
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`) core.info(`WarpBuild mirror cache skipped: ${skipReason}`)
return false return 'off'
} }
core.startGroup('WarpBuild: checkout snapshot cache') core.startGroup('WarpBuild: checkout mirror cache')
try { try {
return await setupInner(settings) return await setupInner(settings)
} catch (error) { } catch (error) {
core.warning( core.warning(
`WarpBuild snapshot cache unavailable, using standard checkout: ${error}` `WarpBuild mirror cache unavailable, using standard checkout: ${error}`
) )
return false return 'off'
} finally { } finally {
core.endGroup() core.endGroup()
} }
} }
async function setupInner(settings: IGitSourceSettings): Promise<boolean> { async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
const sha = settings.commit const refKey = computeRefKey(settings)
plan = {mode: 'off', repoKey, refKey}
// The restore/upload paths shell out to `tar`; without it, fall back cleanly.
if (!(await io.which('tar', false))) {
core.info('tar not found on PATH; using standard checkout')
return false
}
const lookup = await api.lookupSnapshot(repoKey, sha)
const lookup = await api.lookupRestore(repoKey, refKey)
if (lookup.kind === 'disabled') { if (lookup.kind === 'disabled') {
core.info('Snapshot cache is disabled by the backend for this organization') core.info('Mirror cache is disabled by the backend for this organization')
return false return 'off'
} }
if (lookup.kind === 'error') { if (lookup.kind === 'error') {
core.info('Snapshot cache backend unavailable; using standard checkout') core.info('Mirror cache backend unavailable; using standard checkout')
return false return 'off'
} }
if (lookup.kind === 'miss') { if (lookup.kind === 'cold') {
const grant = await api.requestBaseUpload(repoKey)
if (grant.kind === 'grant') {
core.info(
'Cold repo: building the base mirror this run (full fetch, then upload)'
)
plan = {mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url}
return 'cold-build'
}
core.info( core.info(
`Cache miss for ${sha}: the standard fetch will run and its result will be uploaded` 'Cold repo: base build already in progress elsewhere; using standard checkout'
) )
decision = 'miss' return 'off'
return false
} }
core.info( // Restore: seed the base, then this branch's delta if present.
`Cache hit for ${sha}: restoring snapshot (${lookup.info.size_bytes} bytes)` await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS)
) if (lookup.branch) {
if (!(await restoreSnapshot(settings, lookup.info.url, sha))) { try {
return false await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS)
} catch (error) {
core.info(`Branch delta seed skipped (${error}); base only`)
}
} }
core.info('Snapshot restored; skipping the GitHub fetch entirely') core.info(
return true `Seeded base mirror${
lookup.branch ? ' + branch delta' : ''
}; the GitHub fetch will be a tip delta`
)
plan = {mode: 'seeded', repoKey, refKey}
return 'seeded'
} }
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the // Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
// snapshot after the job exits (keeping tar+upload off the customer's billed time). // or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
export async function contribute(settings: IGitSourceSettings): Promise<void> { export async function contribute(settings: IGitSourceSettings): Promise<void> {
if (decision !== 'miss') { try {
if (plan.mode === 'cold-build') {
await uploadBaseMirror(settings)
} else if (plan.mode === 'seeded') {
await uploadBranchDelta(settings)
}
} catch (error) {
core.warning(`WarpBuild mirror upload skipped: ${error}`)
}
}
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
// nothing is extracted into .git directly.
async function seedBundle(
repoPath: string,
url: string,
refNs: string
): Promise<void> {
const tmp = tempBundlePath('seed')
try {
await downloadTo(url, tmp)
await exec.exec('git', [
'-C',
repoPath,
'fetch',
'--quiet',
'--no-tags',
tmp,
`+refs/*:${refNs}/*`
])
} finally {
await fs.promises.rm(tmp, {force: true})
}
}
// Cold-build: after the forced full fetch the repo holds all branches under
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
async function uploadBaseMirror(settings: IGitSourceSettings): Promise<void> {
if (!plan.baseUploadUrl) {
return return
} }
const tmp = tempBundlePath('base')
try { try {
await writeSnapshotManifest(settings) await exec.exec('git', [
} catch (error) { '-C',
core.warning(`Snapshot manifest not written: ${error}`) settings.repositoryPath,
} 'bundle',
} 'create',
tmp,
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` '--remotes=origin',
// component — the tar is remote and extracted into .git, so a crafted member could '--tags'
// escape (e.g. hooks/, ../) and run code during checkout. ])
export async function assertSafeTarMembers(tar: string): Promise<void> { await httpPut(plan.baseUploadUrl, tmp)
let listing = '' core.info('Uploaded base mirror')
await exec.exec('tar', ['-tf', tar], {
silent: true,
listeners: {stdout: (d: Buffer) => (listing += d.toString())}
})
for (const raw of listing.split('\n')) {
const member = raw.trim()
if (!member) {
continue
}
const top = member.replace(/^\.\//, '').split('/')[0]
if (
member.startsWith('/') ||
member.split('/').includes('..') ||
(top !== 'objects' && top !== 'shallow')
) {
throw new Error(`unexpected snapshot tar member: ${member}`)
}
}
}
async function restoreSnapshot(
settings: IGitSourceSettings,
url: string,
sha: string
): Promise<boolean> {
const gitDir = path.join(settings.repositoryPath, '.git')
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-${process.pid}.tar`)
try {
const res = await fetch(url, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
})
if (!res.ok || !res.body) {
throw new Error(`snapshot download answered ${res.status}`)
}
await pipeline(
Readable.fromWeb(res.body as import('stream/web').ReadableStream),
fs.createWriteStream(tmpTar)
)
await assertSafeTarMembers(tmpTar)
await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
const check = await exec.exec(
'git',
['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`],
{ignoreReturnCode: true}
)
if (check !== 0) {
throw new Error(`restored snapshot does not contain ${sha}`)
}
// The ref the skipped fetch would have created; upstream's verification needs it.
const dstRef = computeDestinationRef(settings.ref)
if (dstRef) {
await exec.exec('git', [
'-C',
settings.repositoryPath,
'update-ref',
dstRef,
sha
])
}
return true
} catch (error) {
core.warning(`Snapshot restore failed: ${error}`)
await resetGitObjects(gitDir)
return false
} finally { } finally {
await fs.promises.rm(tmpTar, {force: true}) await fs.promises.rm(tmp, {force: true})
} }
} }
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init` // Seeded: build the base-relative delta (target sha, excluding everything in the base)
// creates, so a failed restore leaves no trace for the fallback fetch. // and overwrite this branch's bundle. Guarded by a per-branch lock server-side.
async function uploadBranchDelta(settings: IGitSourceSettings): Promise<void> {
if (!plan.refKey) {
return // tag / detached HEAD: nothing to roll
}
const repoPath = settings.repositoryPath
const sha = settings.commit
const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha)
if (grant.kind !== 'grant') {
core.info(`Branch delta upload skipped (${grant.kind})`)
return
}
const excludes = await baseRefExcludes(repoPath)
if (excludes.length === 0) {
core.info('No base refs to diff against; branch delta skipped')
return
}
const tmp = tempBundlePath('branch')
try {
await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha])
await exec.exec('git', [
'-C',
repoPath,
'bundle',
'create',
tmp,
UPLOAD_TIP_REF,
...excludes
])
await httpPut(grant.url, tmp)
core.info(`Uploaded branch delta for '${plan.refKey}'`)
} finally {
await fs.promises.rm(tmp, {force: true})
await exec.exec(
'git',
['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF],
{
ignoreReturnCode: true
}
)
}
}
// `^refname` for every seeded base ref — the exclusion set that makes the delta bundle
// base-relative (and therefore order-free).
async function baseRefExcludes(repoPath: string): Promise<string[]> {
let out = ''
await exec.exec(
'git',
['-C', repoPath, 'for-each-ref', '--format=^%(refname)', BASE_REFNS],
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
)
return out
.split('\n')
.map(s => s.trim())
.filter(Boolean)
}
async function downloadTo(url: string, dest: string): Promise<void> {
const res = await fetch(url, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS)
})
if (!res.ok || !res.body) {
throw new Error(`download answered ${res.status}`)
}
await pipeline(
Readable.fromWeb(res.body as import('stream/web').ReadableStream),
fs.createWriteStream(dest)
)
}
async function httpPut(url: string, file: string): Promise<void> {
const stat = await fs.promises.stat(file)
const res = await fetch(url, {
method: 'PUT',
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body
body: fs.createReadStream(file) as any,
duplex: 'half',
headers: {'content-length': String(stat.size)},
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed
} as any)
if (!res.ok) {
throw new Error(`PUT answered ${res.status}`)
}
}
function tempBundlePath(tag: string): string {
return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`)
}
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
export async function resetGitObjects(gitDir: string): Promise<void> { export async function resetGitObjects(gitDir: string): Promise<void> {
await fs.promises.rm(path.join(gitDir, 'objects'), { await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true, recursive: true,
@ -265,30 +349,7 @@ export async function resetGitObjects(gitDir: string): Promise<void> {
}) })
} }
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which // Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`.
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir). export async function hasGit(): Promise<boolean> {
export function manifestDir(): string { return Boolean(await io.which('git', false))
const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
return path.join(runnerTemp, 'wb-snapshots')
}
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(
settings: IGitSourceSettings
): Promise<void> {
const sha = settings.commit
const manifest = {
repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
sha,
git_dir: path.join(settings.repositoryPath, '.git')
}
const dir = manifestDir()
await fs.promises.mkdir(dir, {recursive: true})
await fs.promises.writeFile(
path.join(dir, `${sha}.json`),
JSON.stringify(manifest)
)
core.info(
`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
)
} }