migrate to post job uploads

This commit is contained in:
darshanime 2026-07-09 14:07:20 +05:30
parent 8b5fffd5f3
commit 41e5319596
6 changed files with 183 additions and 253 deletions

View file

@ -2,7 +2,7 @@
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA. objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA.
A later job checking out the same commit restores that snapshot and skips the fetch from A later job checking out the same commit restores that snapshot and skips the fetch from
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.

View file

@ -14,9 +14,11 @@ import {
SKIP_NOT_WARPBUILD, SKIP_NOT_WARPBUILD,
assertSafeTarMembers, assertSafeTarMembers,
computeDestinationRef, computeDestinationRef,
getMirrorCacheSkipReason getMirrorCacheSkipReason,
manifestDir,
resetGitObjects
} from '../src/warpbuild/mirror-cache.js' } from '../src/warpbuild/mirror-cache.js'
import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js' import {lookupSnapshot} from '../src/warpbuild/backend-api.js'
import {IGitSourceSettings} from '../src/git-source-settings.js' import {IGitSourceSettings} from '../src/git-source-settings.js'
const WB_ENV = [ const WB_ENV = [
@ -136,6 +138,60 @@ describe('warpbuild snapshot cache', () => {
}) })
}) })
describe('resetGitObjects', () => {
it('restores the empty git-init object layout, dropping restored junk', async () => {
const gitDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-reset-')
)
try {
// Simulate a partially-restored .git: stray objects, a pack, a shallow file.
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
await fs.promises.writeFile(
path.join(gitDir, 'objects', 'ab'),
'junk loose object'
)
await fs.promises.writeFile(
path.join(gitDir, 'objects', 'pack', 'pack-x.pack'),
'junk pack'
)
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
await resetGitObjects(gitDir)
// shallow gone; objects/ holds only the two empty init dirs.
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack'])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'pack'))
).toEqual([])
expect(
await fs.promises.readdir(path.join(gitDir, 'objects', 'info'))
).toEqual([])
} finally {
await fs.promises.rm(gitDir, {recursive: true, force: true})
}
})
it('is a no-op-safe on an already-clean .git', async () => {
const gitDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), 'wb-reset-')
)
try {
await expect(resetGitObjects(gitDir)).resolves.toBeUndefined()
expect(
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
).toEqual(['info', 'pack'])
} finally {
await fs.promises.rm(gitDir, {recursive: true, force: true})
}
})
})
describe('assertSafeTarMembers', () => { describe('assertSafeTarMembers', () => {
async function makeTar( async function makeTar(
dir: string, dir: string,
@ -235,20 +291,14 @@ describe('warpbuild snapshot cache', () => {
expect(lastUrl).toContain(`sha=${SHA}`) expect(lastUrl).toContain(`sha=${SHA}`)
}) })
it('maps 404 to miss (upload after the stock fetch)', async () => { it('maps 404 to miss (agent uploads after the job)', async () => {
stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss') expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
}) })
it('maps 409 upload responses to locked (another job is uploading)', async () => {
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
})
it('maps 403 to disabled (backend kill switch)', async () => { it('maps 403 to disabled (backend kill switch)', async () => {
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled') expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
}) })
it('maps other statuses and network failures to error', async () => { it('maps other statuses and network failures to error', async () => {
@ -258,15 +308,24 @@ describe('warpbuild snapshot cache', () => {
throw new Error('boom') throw new Error('boom')
}) as typeof fetch }) as typeof fetch
expect((await lookupSnapshot('123', SHA)).kind).toBe('error') expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
expect((await requestUploadURL('123', SHA)).kind).toBe('error') })
})
describe('manifestDir', () => {
const saved = process.env['RUNNER_TEMP']
afterEach(() => {
if (saved === undefined) {
delete process.env['RUNNER_TEMP']
} else {
process.env['RUNNER_TEMP'] = saved
}
}) })
it('maps 200 upload responses to ok', async () => { it('places manifests under RUNNER_TEMP/wb-snapshots', () => {
stubFetch(200, {url: 'https://s3/put'}) process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x')
expect(await requestUploadURL('123', SHA)).toEqual({ expect(manifestDir()).toBe(
kind: 'ok', path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots')
url: 'https://s3/put' )
})
}) })
}) })
}) })

View file

@ -112,6 +112,8 @@ outputs:
description: 'The branch, tag or SHA that was checked out' description: 'The branch, tag or SHA that was checked out'
commit: commit:
description: 'The commit SHA that was checked out' description: 'The commit SHA that was checked out'
cache-hit:
description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)'
runs: runs:
using: node24 using: node24
main: dist/index.js main: dist/index.js

154
dist/index.js vendored
View file

@ -41697,10 +41697,9 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts ;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
/* eslint-disable i18n-text/no-en -- upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // Client for backend-core's git-mirror download-url endpoint, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the // verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is // org; else = fall back. The upload half runs in the warpbuild-agent after the job.
// uploading this snapshot (skip upload); else = fall back.
const API_TIMEOUT_MS = 10_000; const API_TIMEOUT_MS = 10_000;
function backend_api_baseUrl() { function backend_api_baseUrl() {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@ -41732,40 +41731,6 @@ async function lookupSnapshot(repoKey, sha) {
return { kind: 'error' }; return { kind: 'error' };
} }
} }
async function requestUploadURL(repoKey, sha) {
try {
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify({ repo_key: repoKey, sha }),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
});
if (res.status === 200) {
const body = (await res.json());
if (body.url) {
return { kind: 'ok', url: body.url };
}
return { kind: 'error' };
}
if (res.status === 403) {
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
return { kind: 'disabled' };
}
if (res.status === 409) {
core_debug(`[wb-cache] upload-url answered 409 (locked)`);
return { kind: 'locked' };
}
core_debug(`[wb-cache] upload-url answered ${res.status}`);
return { kind: 'error' };
}
catch (error) {
core_debug(`[wb-cache] upload-url failed: ${error}`);
return { kind: 'error' };
}
}
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts ;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ /* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
@ -41782,9 +41747,6 @@ async function requestUploadURL(repoKey, sha) {
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are // produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
// immutable (no expiry). Fail-open: any error degrades to stock behavior. // immutable (no expiry). Fail-open: any error degrades to stock behavior.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000; const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
const CACHE_STATE_FILE = 'wb-cache-state';
// Logged at debug (normal state outside WarpBuild); other reasons log at info. // Logged at debug (normal state outside WarpBuild); other reasons log at info.
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
@ -41849,16 +41811,17 @@ function computeDestinationRef(ref) {
return null; return null;
} }
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. // Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
// Sets the `cache-hit` action output (true only on a restore).
async function setup(settings) { async function setup(settings) {
const restored = await setupImpl(settings);
setOutput('cache-hit', restored ? 'true' : 'false');
return restored;
}
async function setupImpl(settings) {
decision = 'off'; decision = 'off';
const skipReason = getMirrorCacheSkipReason(settings); const skipReason = getMirrorCacheSkipReason(settings);
if (skipReason) { if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) { info(`WarpBuild snapshot cache skipped: ${skipReason}`);
core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
}
else {
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
}
return false; return false;
} }
startGroup('WarpBuild: checkout snapshot cache'); startGroup('WarpBuild: checkout snapshot cache');
@ -41902,20 +41865,17 @@ async function setupInner(settings) {
info('Snapshot restored; skipping the GitHub fetch entirely'); info('Snapshot restored; skipping the GitHub fetch entirely');
return true; return true;
} }
// Runs after checkout; uploads the fetch result on a miss. Failures only warn. // Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
async function contribute(settings) { async function contribute(settings) {
if (decision !== 'miss') { if (decision !== 'miss') {
return; return;
} }
startGroup('WarpBuild: uploading checkout snapshot');
try { try {
await uploadSnapshot(settings); await writeSnapshotManifest(settings);
} }
catch (error) { catch (error) {
warning(`Snapshot upload skipped: ${error}`); warning(`Snapshot manifest not written: ${error}`);
}
finally {
endGroup();
} }
} }
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..` // Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
@ -41968,70 +41928,50 @@ async function restoreSnapshot(settings, url, sha) {
sha sha
]); ]);
} }
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
return true; return true;
} }
catch (error) { catch (error) {
warning(`Snapshot restore failed: ${error}`); warning(`Snapshot restore failed: ${error}`);
// Reset .git to its freshly-init state. await resetGitObjects(gitDir);
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
recursive: true,
force: true
});
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
recursive: true
});
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
recursive: true
});
return false; return false;
} }
finally { finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true }); await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
} }
} }
async function uploadSnapshot(settings) { // Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
const repoKey = process.env['GITHUB_REPOSITORY_ID']; // creates, so a failed restore leaves no trace for the fallback fetch.
async function resetGitObjects(gitDir) {
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
recursive: true,
force: true
});
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
recursive: true
});
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
recursive: true
});
}
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
function manifestDir() {
const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir();
return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots');
}
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(settings) {
const sha = settings.commit; const sha = settings.commit;
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git'); const manifest = {
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`); repo_key: process.env['GITHUB_REPOSITORY_ID'],
try { sha,
const members = ['objects']; git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git')
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) { };
members.push('shallow'); const dir = manifestDir();
} await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true });
await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]); await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest));
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size; info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`);
const upload = await requestUploadURL(repoKey, sha);
if (upload.kind !== 'ok') {
info(upload.kind === 'locked'
? 'Another job is already uploading this snapshot; skipping'
: upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading');
return;
}
const init = {
method: 'PUT',
headers: {
'content-length': String(size),
'content-type': 'application/x-tar'
},
body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)),
duplex: 'half',
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
};
const res = await fetch(upload.url, init);
if (!res.ok) {
throw new Error(`snapshot upload answered ${res.status}`);
}
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
}
finally {
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
}
} }
;// CONCATENATED MODULE: ./src/git-source-provider.ts ;// CONCATENATED MODULE: ./src/git-source-provider.ts

View file

@ -1,10 +1,9 @@
/* eslint-disable i18n-text/no-en -- upstream convention */ /* eslint-disable i18n-text/no-en -- upstream convention */
import * as core from '@actions/core' import * as core from '@actions/core'
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner // Client for backend-core's git-mirror download-url endpoint, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the // verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is // org; else = fall back. The upload half runs in the warpbuild-agent after the job.
// uploading this snapshot (skip upload); else = fall back.
const API_TIMEOUT_MS = 10_000 const API_TIMEOUT_MS = 10_000
@ -20,12 +19,6 @@ export type SnapshotLookup =
| {kind: 'disabled'} | {kind: 'disabled'}
| {kind: 'error'} | {kind: 'error'}
export type UploadURLResult =
| {kind: 'ok'; url: string}
| {kind: 'disabled'}
| {kind: 'locked'}
| {kind: 'error'}
function baseUrl(): string { function baseUrl(): string {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
} }
@ -65,40 +58,3 @@ export async function lookupSnapshot(
return {kind: 'error'} return {kind: 'error'}
} }
} }
export async function requestUploadURL(
repoKey: string,
sha: string
): Promise<UploadURLResult> {
try {
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
method: 'POST',
headers: {
authorization: authHeader(),
'content-type': 'application/json'
},
body: JSON.stringify({repo_key: repoKey, sha}),
signal: AbortSignal.timeout(API_TIMEOUT_MS)
})
if (res.status === 200) {
const body = (await res.json()) as {url?: string}
if (body.url) {
return {kind: 'ok', url: body.url}
}
return {kind: 'error'}
}
if (res.status === 403) {
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
return {kind: 'disabled'}
}
if (res.status === 409) {
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
return {kind: 'locked'}
}
core.debug(`[wb-cache] upload-url answered ${res.status}`)
return {kind: 'error'}
} catch (error) {
core.debug(`[wb-cache] upload-url failed: ${error}`)
return {kind: 'error'}
}
}

View file

@ -15,10 +15,6 @@ import * as api from './backend-api.js'
// immutable (no expiry). Fail-open: any error degrades to stock behavior. // immutable (no expiry). Fail-open: any error degrades to stock behavior.
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000 const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
const UPLOAD_TIMEOUT_MS = 15 * 60_000
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
export const CACHE_STATE_FILE = 'wb-cache-state'
// Logged at debug (normal state outside WarpBuild); other reasons log at info. // Logged at debug (normal state outside WarpBuild); other reasons log at info.
export const SKIP_NOT_WARPBUILD = export const SKIP_NOT_WARPBUILD =
@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null {
} }
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws. // Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
// Sets the `cache-hit` action output (true only on a restore).
export async function setup(settings: IGitSourceSettings): Promise<boolean> { export async function setup(settings: IGitSourceSettings): Promise<boolean> {
const restored = await setupImpl(settings)
core.setOutput('cache-hit', restored ? 'true' : 'false')
return restored
}
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> {
decision = 'off' decision = 'off'
const skipReason = getMirrorCacheSkipReason(settings) const skipReason = getMirrorCacheSkipReason(settings)
if (skipReason) { if (skipReason) {
if (skipReason === SKIP_NOT_WARPBUILD) { core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
} else {
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
}
return false return false
} }
core.startGroup('WarpBuild: checkout snapshot cache') core.startGroup('WarpBuild: checkout snapshot cache')
@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
return true return true
} }
// Runs after checkout; uploads the fetch result on a miss. Failures only warn. // Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
export async function contribute(settings: IGitSourceSettings): Promise<void> { export async function contribute(settings: IGitSourceSettings): Promise<void> {
if (decision !== 'miss') { if (decision !== 'miss') {
return return
} }
core.startGroup('WarpBuild: uploading checkout snapshot')
try { try {
await uploadSnapshot(settings) await writeSnapshotManifest(settings)
} catch (error) { } catch (error) {
core.warning(`Snapshot upload skipped: ${error}`) core.warning(`Snapshot manifest not written: ${error}`)
} finally {
core.endGroup()
} }
} }
@ -242,80 +239,56 @@ async function restoreSnapshot(
]) ])
} }
await fs.promises.writeFile(
path.join(gitDir, CACHE_STATE_FILE),
`hit ${sha}\n`
)
return true return true
} catch (error) { } catch (error) {
core.warning(`Snapshot restore failed: ${error}`) core.warning(`Snapshot restore failed: ${error}`)
// Reset .git to its freshly-init state. await resetGitObjects(gitDir)
await fs.promises.rm(path.join(gitDir, 'objects'), {
recursive: true,
force: true
})
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
recursive: true
})
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
})
return false return false
} finally { } finally {
await fs.promises.rm(tmpTar, {force: true}) await fs.promises.rm(tmpTar, {force: true})
} }
} }
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> { // Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string // creates, so a failed restore leaves no trace for the fallback fetch.
const sha = settings.commit export async function resetGitObjects(gitDir: string): Promise<void> {
const gitDir = path.join(settings.repositoryPath, '.git') await fs.promises.rm(path.join(gitDir, 'objects'), {
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`) recursive: true,
try { force: true
const members = ['objects'] })
if (fs.existsSync(path.join(gitDir, 'shallow'))) { await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
members.push('shallow') await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
} recursive: true
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]) })
const size = (await fs.promises.stat(tmpTar)).size await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
recursive: true
const upload = await api.requestUploadURL(repoKey, sha) })
if (upload.kind !== 'ok') { }
core.info(
upload.kind === 'locked' // Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
? 'Another job is already uploading this snapshot; skipping' // is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
: upload.kind === 'disabled' export function manifestDir(): string {
? 'Snapshot cache is disabled by the backend; not uploading' const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
: 'Snapshot cache backend unavailable; not uploading' return path.join(runnerTemp, 'wb-snapshots')
) }
return
} // One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
async function writeSnapshotManifest(
const init: RequestInit & {duplex: 'half'} = { settings: IGitSourceSettings
method: 'PUT', ): Promise<void> {
headers: { const sha = settings.commit
'content-length': String(size), const manifest = {
'content-type': 'application/x-tar' repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
}, sha,
body: Readable.toWeb( git_dir: path.join(settings.repositoryPath, '.git')
fs.createReadStream(tmpTar) }
) as unknown as ReadableStream, const dir = manifestDir()
duplex: 'half', await fs.promises.mkdir(dir, {recursive: true})
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) await fs.promises.writeFile(
} path.join(dir, `${sha}.json`),
const res = await fetch(upload.url, init) JSON.stringify(manifest)
if (!res.ok) { )
throw new Error(`snapshot upload answered ${res.status}`) core.info(
} `Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
await fs.promises.writeFile( )
path.join(gitDir, CACHE_STATE_FILE),
`uploaded ${sha}\n`
)
core.info(
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
)
} finally {
await fs.promises.rm(tmpTar, {force: true})
}
} }