mirror of
https://github.com/actions/checkout.git
synced 2026-07-18 03:31:27 +08:00
migrate to post job uploads
This commit is contained in:
parent
8b5fffd5f3
commit
41e5319596
6 changed files with 183 additions and 253 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
||||
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
|
||||
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
|
||||
objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA.
|
||||
A later job checking out the same commit restores that snapshot and skips the fetch from
|
||||
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
|
||||
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@ import {
|
|||
SKIP_NOT_WARPBUILD,
|
||||
assertSafeTarMembers,
|
||||
computeDestinationRef,
|
||||
getMirrorCacheSkipReason
|
||||
getMirrorCacheSkipReason,
|
||||
manifestDir,
|
||||
resetGitObjects
|
||||
} from '../src/warpbuild/mirror-cache.js'
|
||||
import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js'
|
||||
import {lookupSnapshot} from '../src/warpbuild/backend-api.js'
|
||||
import {IGitSourceSettings} from '../src/git-source-settings.js'
|
||||
|
||||
const WB_ENV = [
|
||||
|
|
@ -136,6 +138,60 @@ describe('warpbuild snapshot cache', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('resetGitObjects', () => {
|
||||
it('restores the empty git-init object layout, dropping restored junk', async () => {
|
||||
const gitDir = await fs.promises.mkdtemp(
|
||||
path.join(os.tmpdir(), 'wb-reset-')
|
||||
)
|
||||
try {
|
||||
// Simulate a partially-restored .git: stray objects, a pack, a shallow file.
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
})
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, 'objects', 'ab'),
|
||||
'junk loose object'
|
||||
)
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, 'objects', 'pack', 'pack-x.pack'),
|
||||
'junk pack'
|
||||
)
|
||||
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
|
||||
|
||||
await resetGitObjects(gitDir)
|
||||
|
||||
// shallow gone; objects/ holds only the two empty init dirs.
|
||||
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
|
||||
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
|
||||
expect(
|
||||
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
|
||||
).toEqual(['info', 'pack'])
|
||||
expect(
|
||||
await fs.promises.readdir(path.join(gitDir, 'objects', 'pack'))
|
||||
).toEqual([])
|
||||
expect(
|
||||
await fs.promises.readdir(path.join(gitDir, 'objects', 'info'))
|
||||
).toEqual([])
|
||||
} finally {
|
||||
await fs.promises.rm(gitDir, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
|
||||
it('is a no-op-safe on an already-clean .git', async () => {
|
||||
const gitDir = await fs.promises.mkdtemp(
|
||||
path.join(os.tmpdir(), 'wb-reset-')
|
||||
)
|
||||
try {
|
||||
await expect(resetGitObjects(gitDir)).resolves.toBeUndefined()
|
||||
expect(
|
||||
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
|
||||
).toEqual(['info', 'pack'])
|
||||
} finally {
|
||||
await fs.promises.rm(gitDir, {recursive: true, force: true})
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertSafeTarMembers', () => {
|
||||
async function makeTar(
|
||||
dir: string,
|
||||
|
|
@ -235,20 +291,14 @@ describe('warpbuild snapshot cache', () => {
|
|||
expect(lastUrl).toContain(`sha=${SHA}`)
|
||||
})
|
||||
|
||||
it('maps 404 to miss (upload after the stock fetch)', async () => {
|
||||
it('maps 404 to miss (agent uploads after the job)', async () => {
|
||||
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
|
||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
|
||||
})
|
||||
|
||||
it('maps 409 upload responses to locked (another job is uploading)', async () => {
|
||||
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
|
||||
expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
|
||||
})
|
||||
|
||||
it('maps 403 to disabled (backend kill switch)', async () => {
|
||||
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
|
||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
|
||||
expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
|
||||
})
|
||||
|
||||
it('maps other statuses and network failures to error', async () => {
|
||||
|
|
@ -258,15 +308,24 @@ describe('warpbuild snapshot cache', () => {
|
|||
throw new Error('boom')
|
||||
}) as typeof fetch
|
||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
|
||||
expect((await requestUploadURL('123', SHA)).kind).toBe('error')
|
||||
})
|
||||
})
|
||||
|
||||
describe('manifestDir', () => {
|
||||
const saved = process.env['RUNNER_TEMP']
|
||||
afterEach(() => {
|
||||
if (saved === undefined) {
|
||||
delete process.env['RUNNER_TEMP']
|
||||
} else {
|
||||
process.env['RUNNER_TEMP'] = saved
|
||||
}
|
||||
})
|
||||
|
||||
it('maps 200 upload responses to ok', async () => {
|
||||
stubFetch(200, {url: 'https://s3/put'})
|
||||
expect(await requestUploadURL('123', SHA)).toEqual({
|
||||
kind: 'ok',
|
||||
url: 'https://s3/put'
|
||||
})
|
||||
it('places manifests under RUNNER_TEMP/wb-snapshots', () => {
|
||||
process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x')
|
||||
expect(manifestDir()).toBe(
|
||||
path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots')
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -112,6 +112,8 @@ outputs:
|
|||
description: 'The branch, tag or SHA that was checked out'
|
||||
commit:
|
||||
description: 'The commit SHA that was checked out'
|
||||
cache-hit:
|
||||
description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)'
|
||||
runs:
|
||||
using: node24
|
||||
main: dist/index.js
|
||||
|
|
|
|||
154
dist/index.js
vendored
154
dist/index.js
vendored
|
|
@ -41697,10 +41697,9 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
|
|||
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
|
||||
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||
|
||||
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
||||
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
|
||||
// uploading this snapshot (skip upload); else = fall back.
|
||||
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
|
||||
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
|
||||
const API_TIMEOUT_MS = 10_000;
|
||||
function backend_api_baseUrl() {
|
||||
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
|
||||
|
|
@ -41732,40 +41731,6 @@ async function lookupSnapshot(repoKey, sha) {
|
|||
return { kind: 'error' };
|
||||
}
|
||||
}
|
||||
async function requestUploadURL(repoKey, sha) {
|
||||
try {
|
||||
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: authHeader(),
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ repo_key: repoKey, sha }),
|
||||
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||
});
|
||||
if (res.status === 200) {
|
||||
const body = (await res.json());
|
||||
if (body.url) {
|
||||
return { kind: 'ok', url: body.url };
|
||||
}
|
||||
return { kind: 'error' };
|
||||
}
|
||||
if (res.status === 403) {
|
||||
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
|
||||
return { kind: 'disabled' };
|
||||
}
|
||||
if (res.status === 409) {
|
||||
core_debug(`[wb-cache] upload-url answered 409 (locked)`);
|
||||
return { kind: 'locked' };
|
||||
}
|
||||
core_debug(`[wb-cache] upload-url answered ${res.status}`);
|
||||
return { kind: 'error' };
|
||||
}
|
||||
catch (error) {
|
||||
core_debug(`[wb-cache] upload-url failed: ${error}`);
|
||||
return { kind: 'error' };
|
||||
}
|
||||
}
|
||||
|
||||
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
|
||||
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||
|
|
@ -41782,9 +41747,6 @@ async function requestUploadURL(repoKey, sha) {
|
|||
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
|
||||
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
||||
const CACHE_STATE_FILE = 'wb-cache-state';
|
||||
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
|
||||
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
|
||||
|
|
@ -41849,16 +41811,17 @@ function computeDestinationRef(ref) {
|
|||
return null;
|
||||
}
|
||||
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||
// Sets the `cache-hit` action output (true only on a restore).
|
||||
async function setup(settings) {
|
||||
const restored = await setupImpl(settings);
|
||||
setOutput('cache-hit', restored ? 'true' : 'false');
|
||||
return restored;
|
||||
}
|
||||
async function setupImpl(settings) {
|
||||
decision = 'off';
|
||||
const skipReason = getMirrorCacheSkipReason(settings);
|
||||
if (skipReason) {
|
||||
if (skipReason === SKIP_NOT_WARPBUILD) {
|
||||
core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||
}
|
||||
else {
|
||||
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||
}
|
||||
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||
return false;
|
||||
}
|
||||
startGroup('WarpBuild: checkout snapshot cache');
|
||||
|
|
@ -41902,20 +41865,17 @@ async function setupInner(settings) {
|
|||
info('Snapshot restored; skipping the GitHub fetch entirely');
|
||||
return true;
|
||||
}
|
||||
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
||||
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
|
||||
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
|
||||
async function contribute(settings) {
|
||||
if (decision !== 'miss') {
|
||||
return;
|
||||
}
|
||||
startGroup('WarpBuild: uploading checkout snapshot');
|
||||
try {
|
||||
await uploadSnapshot(settings);
|
||||
await writeSnapshotManifest(settings);
|
||||
}
|
||||
catch (error) {
|
||||
warning(`Snapshot upload skipped: ${error}`);
|
||||
}
|
||||
finally {
|
||||
endGroup();
|
||||
warning(`Snapshot manifest not written: ${error}`);
|
||||
}
|
||||
}
|
||||
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
|
||||
|
|
@ -41968,70 +41928,50 @@ async function restoreSnapshot(settings, url, sha) {
|
|||
sha
|
||||
]);
|
||||
}
|
||||
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
|
||||
return true;
|
||||
}
|
||||
catch (error) {
|
||||
warning(`Snapshot restore failed: ${error}`);
|
||||
// Reset .git to its freshly-init state.
|
||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
});
|
||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
|
||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
});
|
||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
});
|
||||
await resetGitObjects(gitDir);
|
||||
return false;
|
||||
}
|
||||
finally {
|
||||
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
||||
}
|
||||
}
|
||||
async function uploadSnapshot(settings) {
|
||||
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
|
||||
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
|
||||
// creates, so a failed restore leaves no trace for the fallback fetch.
|
||||
async function resetGitObjects(gitDir) {
|
||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
});
|
||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
|
||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
});
|
||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
});
|
||||
}
|
||||
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
|
||||
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
|
||||
function manifestDir() {
|
||||
const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir();
|
||||
return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots');
|
||||
}
|
||||
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
|
||||
async function writeSnapshotManifest(settings) {
|
||||
const sha = settings.commit;
|
||||
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
|
||||
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`);
|
||||
try {
|
||||
const members = ['objects'];
|
||||
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) {
|
||||
members.push('shallow');
|
||||
}
|
||||
await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]);
|
||||
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
|
||||
const upload = await requestUploadURL(repoKey, sha);
|
||||
if (upload.kind !== 'ok') {
|
||||
info(upload.kind === 'locked'
|
||||
? 'Another job is already uploading this snapshot; skipping'
|
||||
: upload.kind === 'disabled'
|
||||
? 'Snapshot cache is disabled by the backend; not uploading'
|
||||
: 'Snapshot cache backend unavailable; not uploading');
|
||||
return;
|
||||
}
|
||||
const init = {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'content-length': String(size),
|
||||
'content-type': 'application/x-tar'
|
||||
},
|
||||
body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)),
|
||||
duplex: 'half',
|
||||
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||
};
|
||||
const res = await fetch(upload.url, init);
|
||||
if (!res.ok) {
|
||||
throw new Error(`snapshot upload answered ${res.status}`);
|
||||
}
|
||||
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
|
||||
info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
|
||||
}
|
||||
finally {
|
||||
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
||||
}
|
||||
const manifest = {
|
||||
repo_key: process.env['GITHUB_REPOSITORY_ID'],
|
||||
sha,
|
||||
git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git')
|
||||
};
|
||||
const dir = manifestDir();
|
||||
await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true });
|
||||
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest));
|
||||
info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`);
|
||||
}
|
||||
|
||||
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||
import * as core from '@actions/core'
|
||||
|
||||
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
||||
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
|
||||
// uploading this snapshot (skip upload); else = fall back.
|
||||
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
|
||||
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
|
||||
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
|
||||
|
||||
const API_TIMEOUT_MS = 10_000
|
||||
|
||||
|
|
@ -20,12 +19,6 @@ export type SnapshotLookup =
|
|||
| {kind: 'disabled'}
|
||||
| {kind: 'error'}
|
||||
|
||||
export type UploadURLResult =
|
||||
| {kind: 'ok'; url: string}
|
||||
| {kind: 'disabled'}
|
||||
| {kind: 'locked'}
|
||||
| {kind: 'error'}
|
||||
|
||||
function baseUrl(): string {
|
||||
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
||||
}
|
||||
|
|
@ -65,40 +58,3 @@ export async function lookupSnapshot(
|
|||
return {kind: 'error'}
|
||||
}
|
||||
}
|
||||
|
||||
export async function requestUploadURL(
|
||||
repoKey: string,
|
||||
sha: string
|
||||
): Promise<UploadURLResult> {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: authHeader(),
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({repo_key: repoKey, sha}),
|
||||
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||
})
|
||||
if (res.status === 200) {
|
||||
const body = (await res.json()) as {url?: string}
|
||||
if (body.url) {
|
||||
return {kind: 'ok', url: body.url}
|
||||
}
|
||||
return {kind: 'error'}
|
||||
}
|
||||
if (res.status === 403) {
|
||||
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
|
||||
return {kind: 'disabled'}
|
||||
}
|
||||
if (res.status === 409) {
|
||||
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
|
||||
return {kind: 'locked'}
|
||||
}
|
||||
core.debug(`[wb-cache] upload-url answered ${res.status}`)
|
||||
return {kind: 'error'}
|
||||
} catch (error) {
|
||||
core.debug(`[wb-cache] upload-url failed: ${error}`)
|
||||
return {kind: 'error'}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,10 +15,6 @@ import * as api from './backend-api.js'
|
|||
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||
|
||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
||||
const UPLOAD_TIMEOUT_MS = 15 * 60_000
|
||||
|
||||
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
||||
export const CACHE_STATE_FILE = 'wb-cache-state'
|
||||
|
||||
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||
export const SKIP_NOT_WARPBUILD =
|
||||
|
|
@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null {
|
|||
}
|
||||
|
||||
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||
// Sets the `cache-hit` action output (true only on a restore).
|
||||
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
|
||||
const restored = await setupImpl(settings)
|
||||
core.setOutput('cache-hit', restored ? 'true' : 'false')
|
||||
return restored
|
||||
}
|
||||
|
||||
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> {
|
||||
decision = 'off'
|
||||
const skipReason = getMirrorCacheSkipReason(settings)
|
||||
if (skipReason) {
|
||||
if (skipReason === SKIP_NOT_WARPBUILD) {
|
||||
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
} else {
|
||||
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
}
|
||||
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||
return false
|
||||
}
|
||||
core.startGroup('WarpBuild: checkout snapshot cache')
|
||||
|
|
@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
|
|||
return true
|
||||
}
|
||||
|
||||
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
||||
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
|
||||
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
|
||||
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||
if (decision !== 'miss') {
|
||||
return
|
||||
}
|
||||
core.startGroup('WarpBuild: uploading checkout snapshot')
|
||||
try {
|
||||
await uploadSnapshot(settings)
|
||||
await writeSnapshotManifest(settings)
|
||||
} catch (error) {
|
||||
core.warning(`Snapshot upload skipped: ${error}`)
|
||||
} finally {
|
||||
core.endGroup()
|
||||
core.warning(`Snapshot manifest not written: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,80 +239,56 @@ async function restoreSnapshot(
|
|||
])
|
||||
}
|
||||
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, CACHE_STATE_FILE),
|
||||
`hit ${sha}\n`
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
core.warning(`Snapshot restore failed: ${error}`)
|
||||
// Reset .git to its freshly-init state.
|
||||
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
})
|
||||
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
})
|
||||
await resetGitObjects(gitDir)
|
||||
return false
|
||||
} finally {
|
||||
await fs.promises.rm(tmpTar, {force: true})
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
|
||||
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
||||
const sha = settings.commit
|
||||
const gitDir = path.join(settings.repositoryPath, '.git')
|
||||
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
|
||||
try {
|
||||
const members = ['objects']
|
||||
if (fs.existsSync(path.join(gitDir, 'shallow'))) {
|
||||
members.push('shallow')
|
||||
}
|
||||
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
|
||||
const size = (await fs.promises.stat(tmpTar)).size
|
||||
|
||||
const upload = await api.requestUploadURL(repoKey, sha)
|
||||
if (upload.kind !== 'ok') {
|
||||
core.info(
|
||||
upload.kind === 'locked'
|
||||
? 'Another job is already uploading this snapshot; skipping'
|
||||
: upload.kind === 'disabled'
|
||||
? 'Snapshot cache is disabled by the backend; not uploading'
|
||||
: 'Snapshot cache backend unavailable; not uploading'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const init: RequestInit & {duplex: 'half'} = {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'content-length': String(size),
|
||||
'content-type': 'application/x-tar'
|
||||
},
|
||||
body: Readable.toWeb(
|
||||
fs.createReadStream(tmpTar)
|
||||
) as unknown as ReadableStream,
|
||||
duplex: 'half',
|
||||
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||
}
|
||||
const res = await fetch(upload.url, init)
|
||||
if (!res.ok) {
|
||||
throw new Error(`snapshot upload answered ${res.status}`)
|
||||
}
|
||||
await fs.promises.writeFile(
|
||||
path.join(gitDir, CACHE_STATE_FILE),
|
||||
`uploaded ${sha}\n`
|
||||
)
|
||||
core.info(
|
||||
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
|
||||
)
|
||||
} finally {
|
||||
await fs.promises.rm(tmpTar, {force: true})
|
||||
}
|
||||
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
|
||||
// creates, so a failed restore leaves no trace for the fallback fetch.
|
||||
export async function resetGitObjects(gitDir: string): Promise<void> {
|
||||
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||
recursive: true,
|
||||
force: true
|
||||
})
|
||||
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||
recursive: true
|
||||
})
|
||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||
recursive: true
|
||||
})
|
||||
}
|
||||
|
||||
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
|
||||
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
|
||||
export function manifestDir(): string {
|
||||
const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
|
||||
return path.join(runnerTemp, 'wb-snapshots')
|
||||
}
|
||||
|
||||
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
|
||||
async function writeSnapshotManifest(
|
||||
settings: IGitSourceSettings
|
||||
): Promise<void> {
|
||||
const sha = settings.commit
|
||||
const manifest = {
|
||||
repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
|
||||
sha,
|
||||
git_dir: path.join(settings.repositoryPath, '.git')
|
||||
}
|
||||
const dir = manifestDir()
|
||||
await fs.promises.mkdir(dir, {recursive: true})
|
||||
await fs.promises.writeFile(
|
||||
path.join(dir, `${sha}.json`),
|
||||
JSON.stringify(manifest)
|
||||
)
|
||||
core.info(
|
||||
`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue