mirror of
https://github.com/actions/checkout.git
synced 2026-07-18 03:50:07 +08:00
migrate to post job uploads
This commit is contained in:
parent
8b5fffd5f3
commit
41e5319596
6 changed files with 183 additions and 253 deletions
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
||||||
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
|
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
|
||||||
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
|
objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA.
|
||||||
A later job checking out the same commit restores that snapshot and skips the fetch from
|
A later job checking out the same commit restores that snapshot and skips the fetch from
|
||||||
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
|
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
|
||||||
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
|
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
|
||||||
|
|
|
||||||
|
|
@ -14,9 +14,11 @@ import {
|
||||||
SKIP_NOT_WARPBUILD,
|
SKIP_NOT_WARPBUILD,
|
||||||
assertSafeTarMembers,
|
assertSafeTarMembers,
|
||||||
computeDestinationRef,
|
computeDestinationRef,
|
||||||
getMirrorCacheSkipReason
|
getMirrorCacheSkipReason,
|
||||||
|
manifestDir,
|
||||||
|
resetGitObjects
|
||||||
} from '../src/warpbuild/mirror-cache.js'
|
} from '../src/warpbuild/mirror-cache.js'
|
||||||
import {lookupSnapshot, requestUploadURL} from '../src/warpbuild/backend-api.js'
|
import {lookupSnapshot} from '../src/warpbuild/backend-api.js'
|
||||||
import {IGitSourceSettings} from '../src/git-source-settings.js'
|
import {IGitSourceSettings} from '../src/git-source-settings.js'
|
||||||
|
|
||||||
const WB_ENV = [
|
const WB_ENV = [
|
||||||
|
|
@ -136,6 +138,60 @@ describe('warpbuild snapshot cache', () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('resetGitObjects', () => {
|
||||||
|
it('restores the empty git-init object layout, dropping restored junk', async () => {
|
||||||
|
const gitDir = await fs.promises.mkdtemp(
|
||||||
|
path.join(os.tmpdir(), 'wb-reset-')
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
// Simulate a partially-restored .git: stray objects, a pack, a shallow file.
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
path.join(gitDir, 'objects', 'ab'),
|
||||||
|
'junk loose object'
|
||||||
|
)
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
path.join(gitDir, 'objects', 'pack', 'pack-x.pack'),
|
||||||
|
'junk pack'
|
||||||
|
)
|
||||||
|
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
|
||||||
|
|
||||||
|
await resetGitObjects(gitDir)
|
||||||
|
|
||||||
|
// shallow gone; objects/ holds only the two empty init dirs.
|
||||||
|
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
|
||||||
|
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
|
||||||
|
expect(
|
||||||
|
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
|
||||||
|
).toEqual(['info', 'pack'])
|
||||||
|
expect(
|
||||||
|
await fs.promises.readdir(path.join(gitDir, 'objects', 'pack'))
|
||||||
|
).toEqual([])
|
||||||
|
expect(
|
||||||
|
await fs.promises.readdir(path.join(gitDir, 'objects', 'info'))
|
||||||
|
).toEqual([])
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(gitDir, {recursive: true, force: true})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is a no-op-safe on an already-clean .git', async () => {
|
||||||
|
const gitDir = await fs.promises.mkdtemp(
|
||||||
|
path.join(os.tmpdir(), 'wb-reset-')
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
await expect(resetGitObjects(gitDir)).resolves.toBeUndefined()
|
||||||
|
expect(
|
||||||
|
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
|
||||||
|
).toEqual(['info', 'pack'])
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(gitDir, {recursive: true, force: true})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('assertSafeTarMembers', () => {
|
describe('assertSafeTarMembers', () => {
|
||||||
async function makeTar(
|
async function makeTar(
|
||||||
dir: string,
|
dir: string,
|
||||||
|
|
@ -235,20 +291,14 @@ describe('warpbuild snapshot cache', () => {
|
||||||
expect(lastUrl).toContain(`sha=${SHA}`)
|
expect(lastUrl).toContain(`sha=${SHA}`)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('maps 404 to miss (upload after the stock fetch)', async () => {
|
it('maps 404 to miss (agent uploads after the job)', async () => {
|
||||||
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
|
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
|
||||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('maps 409 upload responses to locked (another job is uploading)', async () => {
|
|
||||||
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
|
|
||||||
expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
|
|
||||||
})
|
|
||||||
|
|
||||||
it('maps 403 to disabled (backend kill switch)', async () => {
|
it('maps 403 to disabled (backend kill switch)', async () => {
|
||||||
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
|
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
|
||||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')
|
||||||
expect((await requestUploadURL('123', SHA)).kind).toBe('disabled')
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it('maps other statuses and network failures to error', async () => {
|
it('maps other statuses and network failures to error', async () => {
|
||||||
|
|
@ -258,15 +308,24 @@ describe('warpbuild snapshot cache', () => {
|
||||||
throw new Error('boom')
|
throw new Error('boom')
|
||||||
}) as typeof fetch
|
}) as typeof fetch
|
||||||
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
|
expect((await lookupSnapshot('123', SHA)).kind).toBe('error')
|
||||||
expect((await requestUploadURL('123', SHA)).kind).toBe('error')
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('manifestDir', () => {
|
||||||
|
const saved = process.env['RUNNER_TEMP']
|
||||||
|
afterEach(() => {
|
||||||
|
if (saved === undefined) {
|
||||||
|
delete process.env['RUNNER_TEMP']
|
||||||
|
} else {
|
||||||
|
process.env['RUNNER_TEMP'] = saved
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('maps 200 upload responses to ok', async () => {
|
it('places manifests under RUNNER_TEMP/wb-snapshots', () => {
|
||||||
stubFetch(200, {url: 'https://s3/put'})
|
process.env['RUNNER_TEMP'] = path.join(os.tmpdir(), 'runner-temp-x')
|
||||||
expect(await requestUploadURL('123', SHA)).toEqual({
|
expect(manifestDir()).toBe(
|
||||||
kind: 'ok',
|
path.join(os.tmpdir(), 'runner-temp-x', 'wb-snapshots')
|
||||||
url: 'https://s3/put'
|
)
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -112,6 +112,8 @@ outputs:
|
||||||
description: 'The branch, tag or SHA that was checked out'
|
description: 'The branch, tag or SHA that was checked out'
|
||||||
commit:
|
commit:
|
||||||
description: 'The commit SHA that was checked out'
|
description: 'The commit SHA that was checked out'
|
||||||
|
cache-hit:
|
||||||
|
description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)'
|
||||||
runs:
|
runs:
|
||||||
using: node24
|
using: node24
|
||||||
main: dist/index.js
|
main: dist/index.js
|
||||||
|
|
|
||||||
154
dist/index.js
vendored
154
dist/index.js
vendored
|
|
@ -41697,10 +41697,9 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
|
||||||
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
|
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
|
||||||
/* eslint-disable i18n-text/no-en -- upstream convention */
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
|
|
||||||
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
|
||||||
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
|
||||||
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
|
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
|
||||||
// uploading this snapshot (skip upload); else = fall back.
|
|
||||||
const API_TIMEOUT_MS = 10_000;
|
const API_TIMEOUT_MS = 10_000;
|
||||||
function backend_api_baseUrl() {
|
function backend_api_baseUrl() {
|
||||||
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
|
||||||
|
|
@ -41732,40 +41731,6 @@ async function lookupSnapshot(repoKey, sha) {
|
||||||
return { kind: 'error' };
|
return { kind: 'error' };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async function requestUploadURL(repoKey, sha) {
|
|
||||||
try {
|
|
||||||
const res = await fetch(`${backend_api_baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
authorization: authHeader(),
|
|
||||||
'content-type': 'application/json'
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ repo_key: repoKey, sha }),
|
|
||||||
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
|
||||||
});
|
|
||||||
if (res.status === 200) {
|
|
||||||
const body = (await res.json());
|
|
||||||
if (body.url) {
|
|
||||||
return { kind: 'ok', url: body.url };
|
|
||||||
}
|
|
||||||
return { kind: 'error' };
|
|
||||||
}
|
|
||||||
if (res.status === 403) {
|
|
||||||
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
|
|
||||||
return { kind: 'disabled' };
|
|
||||||
}
|
|
||||||
if (res.status === 409) {
|
|
||||||
core_debug(`[wb-cache] upload-url answered 409 (locked)`);
|
|
||||||
return { kind: 'locked' };
|
|
||||||
}
|
|
||||||
core_debug(`[wb-cache] upload-url answered ${res.status}`);
|
|
||||||
return { kind: 'error' };
|
|
||||||
}
|
|
||||||
catch (error) {
|
|
||||||
core_debug(`[wb-cache] upload-url failed: ${error}`);
|
|
||||||
return { kind: 'error' };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
|
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
|
||||||
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||||
|
|
@ -41782,9 +41747,6 @@ async function requestUploadURL(repoKey, sha) {
|
||||||
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
|
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
|
||||||
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
|
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||||
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
|
|
||||||
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
|
||||||
const CACHE_STATE_FILE = 'wb-cache-state';
|
|
||||||
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||||
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
|
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
|
||||||
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
|
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
|
||||||
|
|
@ -41849,16 +41811,17 @@ function computeDestinationRef(ref) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||||
|
// Sets the `cache-hit` action output (true only on a restore).
|
||||||
async function setup(settings) {
|
async function setup(settings) {
|
||||||
|
const restored = await setupImpl(settings);
|
||||||
|
setOutput('cache-hit', restored ? 'true' : 'false');
|
||||||
|
return restored;
|
||||||
|
}
|
||||||
|
async function setupImpl(settings) {
|
||||||
decision = 'off';
|
decision = 'off';
|
||||||
const skipReason = getMirrorCacheSkipReason(settings);
|
const skipReason = getMirrorCacheSkipReason(settings);
|
||||||
if (skipReason) {
|
if (skipReason) {
|
||||||
if (skipReason === SKIP_NOT_WARPBUILD) {
|
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
||||||
core_debug(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
info(`WarpBuild snapshot cache skipped: ${skipReason}`);
|
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
startGroup('WarpBuild: checkout snapshot cache');
|
startGroup('WarpBuild: checkout snapshot cache');
|
||||||
|
|
@ -41902,20 +41865,17 @@ async function setupInner(settings) {
|
||||||
info('Snapshot restored; skipping the GitHub fetch entirely');
|
info('Snapshot restored; skipping the GitHub fetch entirely');
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
|
||||||
|
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
|
||||||
async function contribute(settings) {
|
async function contribute(settings) {
|
||||||
if (decision !== 'miss') {
|
if (decision !== 'miss') {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
startGroup('WarpBuild: uploading checkout snapshot');
|
|
||||||
try {
|
try {
|
||||||
await uploadSnapshot(settings);
|
await writeSnapshotManifest(settings);
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
warning(`Snapshot upload skipped: ${error}`);
|
warning(`Snapshot manifest not written: ${error}`);
|
||||||
}
|
|
||||||
finally {
|
|
||||||
endGroup();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
|
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
|
||||||
|
|
@ -41968,70 +41928,50 @@ async function restoreSnapshot(settings, url, sha) {
|
||||||
sha
|
sha
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `hit ${sha}\n`);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
warning(`Snapshot restore failed: ${error}`);
|
warning(`Snapshot restore failed: ${error}`);
|
||||||
// Reset .git to its freshly-init state.
|
await resetGitObjects(gitDir);
|
||||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
|
|
||||||
recursive: true,
|
|
||||||
force: true
|
|
||||||
});
|
|
||||||
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
|
|
||||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
|
|
||||||
recursive: true
|
|
||||||
});
|
|
||||||
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
|
|
||||||
recursive: true
|
|
||||||
});
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async function uploadSnapshot(settings) {
|
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
|
||||||
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
|
// creates, so a failed restore leaves no trace for the fallback fetch.
|
||||||
|
async function resetGitObjects(gitDir) {
|
||||||
|
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'objects'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true
|
||||||
|
});
|
||||||
|
await external_fs_namespaceObject.promises.rm(external_path_namespaceObject.join(gitDir, 'shallow'), { force: true });
|
||||||
|
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'info'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
await external_fs_namespaceObject.promises.mkdir(external_path_namespaceObject.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
|
||||||
|
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
|
||||||
|
function manifestDir() {
|
||||||
|
const runnerTemp = process.env['RUNNER_TEMP'] || external_os_namespaceObject.tmpdir();
|
||||||
|
return external_path_namespaceObject.join(runnerTemp, 'wb-snapshots');
|
||||||
|
}
|
||||||
|
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
|
||||||
|
async function writeSnapshotManifest(settings) {
|
||||||
const sha = settings.commit;
|
const sha = settings.commit;
|
||||||
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
|
const manifest = {
|
||||||
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-up-${process.pid}.tar`);
|
repo_key: process.env['GITHUB_REPOSITORY_ID'],
|
||||||
try {
|
sha,
|
||||||
const members = ['objects'];
|
git_dir: external_path_namespaceObject.join(settings.repositoryPath, '.git')
|
||||||
if (external_fs_namespaceObject.existsSync(external_path_namespaceObject.join(gitDir, 'shallow'))) {
|
};
|
||||||
members.push('shallow');
|
const dir = manifestDir();
|
||||||
}
|
await external_fs_namespaceObject.promises.mkdir(dir, { recursive: true });
|
||||||
await exec_exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members]);
|
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(dir, `${sha}.json`), JSON.stringify(manifest));
|
||||||
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
|
info(`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`);
|
||||||
const upload = await requestUploadURL(repoKey, sha);
|
|
||||||
if (upload.kind !== 'ok') {
|
|
||||||
info(upload.kind === 'locked'
|
|
||||||
? 'Another job is already uploading this snapshot; skipping'
|
|
||||||
: upload.kind === 'disabled'
|
|
||||||
? 'Snapshot cache is disabled by the backend; not uploading'
|
|
||||||
: 'Snapshot cache backend unavailable; not uploading');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const init = {
|
|
||||||
method: 'PUT',
|
|
||||||
headers: {
|
|
||||||
'content-length': String(size),
|
|
||||||
'content-type': 'application/x-tar'
|
|
||||||
},
|
|
||||||
body: external_stream_namespaceObject.Readable.toWeb(external_fs_namespaceObject.createReadStream(tmpTar)),
|
|
||||||
duplex: 'half',
|
|
||||||
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
|
||||||
};
|
|
||||||
const res = await fetch(upload.url, init);
|
|
||||||
if (!res.ok) {
|
|
||||||
throw new Error(`snapshot upload answered ${res.status}`);
|
|
||||||
}
|
|
||||||
await external_fs_namespaceObject.promises.writeFile(external_path_namespaceObject.join(gitDir, CACHE_STATE_FILE), `uploaded ${sha}\n`);
|
|
||||||
info(`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`);
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
await external_fs_namespaceObject.promises.rm(tmpTar, { force: true });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
/* eslint-disable i18n-text/no-en -- upstream convention */
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
import * as core from '@actions/core'
|
import * as core from '@actions/core'
|
||||||
|
|
||||||
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
|
// Client for backend-core's git-mirror download-url endpoint, authed by the runner
|
||||||
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
|
// verification token. Contract: 200 = presigned URL (hit); 404 = miss; 403 = unservable
|
||||||
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
|
// org; else = fall back. The upload half runs in the warpbuild-agent after the job.
|
||||||
// uploading this snapshot (skip upload); else = fall back.
|
|
||||||
|
|
||||||
const API_TIMEOUT_MS = 10_000
|
const API_TIMEOUT_MS = 10_000
|
||||||
|
|
||||||
|
|
@ -20,12 +19,6 @@ export type SnapshotLookup =
|
||||||
| {kind: 'disabled'}
|
| {kind: 'disabled'}
|
||||||
| {kind: 'error'}
|
| {kind: 'error'}
|
||||||
|
|
||||||
export type UploadURLResult =
|
|
||||||
| {kind: 'ok'; url: string}
|
|
||||||
| {kind: 'disabled'}
|
|
||||||
| {kind: 'locked'}
|
|
||||||
| {kind: 'error'}
|
|
||||||
|
|
||||||
function baseUrl(): string {
|
function baseUrl(): string {
|
||||||
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
||||||
}
|
}
|
||||||
|
|
@ -65,40 +58,3 @@ export async function lookupSnapshot(
|
||||||
return {kind: 'error'}
|
return {kind: 'error'}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function requestUploadURL(
|
|
||||||
repoKey: string,
|
|
||||||
sha: string
|
|
||||||
): Promise<UploadURLResult> {
|
|
||||||
try {
|
|
||||||
const res = await fetch(`${baseUrl()}/api/v1/git-mirrors/upload-url`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
authorization: authHeader(),
|
|
||||||
'content-type': 'application/json'
|
|
||||||
},
|
|
||||||
body: JSON.stringify({repo_key: repoKey, sha}),
|
|
||||||
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
|
||||||
})
|
|
||||||
if (res.status === 200) {
|
|
||||||
const body = (await res.json()) as {url?: string}
|
|
||||||
if (body.url) {
|
|
||||||
return {kind: 'ok', url: body.url}
|
|
||||||
}
|
|
||||||
return {kind: 'error'}
|
|
||||||
}
|
|
||||||
if (res.status === 403) {
|
|
||||||
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
|
|
||||||
return {kind: 'disabled'}
|
|
||||||
}
|
|
||||||
if (res.status === 409) {
|
|
||||||
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
|
|
||||||
return {kind: 'locked'}
|
|
||||||
}
|
|
||||||
core.debug(`[wb-cache] upload-url answered ${res.status}`)
|
|
||||||
return {kind: 'error'}
|
|
||||||
} catch (error) {
|
|
||||||
core.debug(`[wb-cache] upload-url failed: ${error}`)
|
|
||||||
return {kind: 'error'}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -15,10 +15,6 @@ import * as api from './backend-api.js'
|
||||||
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
|
||||||
|
|
||||||
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
const DOWNLOAD_TIMEOUT_MS = 15 * 60_000
|
||||||
const UPLOAD_TIMEOUT_MS = 15 * 60_000
|
|
||||||
|
|
||||||
// "hit <sha>" | "uploaded <sha>", for e2e assertions.
|
|
||||||
export const CACHE_STATE_FILE = 'wb-cache-state'
|
|
||||||
|
|
||||||
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
// Logged at debug (normal state outside WarpBuild); other reasons log at info.
|
||||||
export const SKIP_NOT_WARPBUILD =
|
export const SKIP_NOT_WARPBUILD =
|
||||||
|
|
@ -97,15 +93,18 @@ export function computeDestinationRef(ref: string): string | null {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
// Runs right after `git init`; true = restored (caller skips the fetch). Never throws.
|
||||||
|
// Sets the `cache-hit` action output (true only on a restore).
|
||||||
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
|
export async function setup(settings: IGitSourceSettings): Promise<boolean> {
|
||||||
|
const restored = await setupImpl(settings)
|
||||||
|
core.setOutput('cache-hit', restored ? 'true' : 'false')
|
||||||
|
return restored
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setupImpl(settings: IGitSourceSettings): Promise<boolean> {
|
||||||
decision = 'off'
|
decision = 'off'
|
||||||
const skipReason = getMirrorCacheSkipReason(settings)
|
const skipReason = getMirrorCacheSkipReason(settings)
|
||||||
if (skipReason) {
|
if (skipReason) {
|
||||||
if (skipReason === SKIP_NOT_WARPBUILD) {
|
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
||||||
core.debug(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
|
||||||
} else {
|
|
||||||
core.info(`WarpBuild snapshot cache skipped: ${skipReason}`)
|
|
||||||
}
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
core.startGroup('WarpBuild: checkout snapshot cache')
|
core.startGroup('WarpBuild: checkout snapshot cache')
|
||||||
|
|
@ -160,18 +159,16 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs after checkout; uploads the fetch result on a miss. Failures only warn.
|
// Runs after checkout; on a miss records a manifest so the WarpBuild agent uploads the
|
||||||
|
// snapshot after the job exits (keeping tar+upload off the customer's billed time).
|
||||||
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||||
if (decision !== 'miss') {
|
if (decision !== 'miss') {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
core.startGroup('WarpBuild: uploading checkout snapshot')
|
|
||||||
try {
|
try {
|
||||||
await uploadSnapshot(settings)
|
await writeSnapshotManifest(settings)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.warning(`Snapshot upload skipped: ${error}`)
|
core.warning(`Snapshot manifest not written: ${error}`)
|
||||||
} finally {
|
|
||||||
core.endGroup()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -242,80 +239,56 @@ async function restoreSnapshot(
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
await fs.promises.writeFile(
|
|
||||||
path.join(gitDir, CACHE_STATE_FILE),
|
|
||||||
`hit ${sha}\n`
|
|
||||||
)
|
|
||||||
return true
|
return true
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.warning(`Snapshot restore failed: ${error}`)
|
core.warning(`Snapshot restore failed: ${error}`)
|
||||||
// Reset .git to its freshly-init state.
|
await resetGitObjects(gitDir)
|
||||||
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
|
||||||
recursive: true,
|
|
||||||
force: true
|
|
||||||
})
|
|
||||||
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
|
||||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
|
||||||
recursive: true
|
|
||||||
})
|
|
||||||
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
|
||||||
recursive: true
|
|
||||||
})
|
|
||||||
return false
|
return false
|
||||||
} finally {
|
} finally {
|
||||||
await fs.promises.rm(tmpTar, {force: true})
|
await fs.promises.rm(tmpTar, {force: true})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
|
// Reset .git/objects (and drop any restored shallow) to the empty shape `git init`
|
||||||
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
// creates, so a failed restore leaves no trace for the fallback fetch.
|
||||||
const sha = settings.commit
|
export async function resetGitObjects(gitDir: string): Promise<void> {
|
||||||
const gitDir = path.join(settings.repositoryPath, '.git')
|
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||||
const tmpTar = path.join(os.tmpdir(), `wb-snapshot-up-${process.pid}.tar`)
|
recursive: true,
|
||||||
try {
|
force: true
|
||||||
const members = ['objects']
|
})
|
||||||
if (fs.existsSync(path.join(gitDir, 'shallow'))) {
|
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||||
members.push('shallow')
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||||
}
|
recursive: true
|
||||||
await exec.exec('tar', ['-cf', tmpTar, '-C', gitDir, ...members])
|
})
|
||||||
const size = (await fs.promises.stat(tmpTar)).size
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
const upload = await api.requestUploadURL(repoKey, sha)
|
})
|
||||||
if (upload.kind !== 'ok') {
|
}
|
||||||
core.info(
|
|
||||||
upload.kind === 'locked'
|
// Directory the agent scans post-job for snapshots to upload. Under RUNNER_TEMP, which
|
||||||
? 'Another job is already uploading this snapshot; skipping'
|
// is <runner>/_work/_temp — host-visible to the agent (it knows the runner dir).
|
||||||
: upload.kind === 'disabled'
|
export function manifestDir(): string {
|
||||||
? 'Snapshot cache is disabled by the backend; not uploading'
|
const runnerTemp = process.env['RUNNER_TEMP'] || os.tmpdir()
|
||||||
: 'Snapshot cache backend unavailable; not uploading'
|
return path.join(runnerTemp, 'wb-snapshots')
|
||||||
)
|
}
|
||||||
return
|
|
||||||
}
|
// One manifest per cache miss: the agent tars git_dir/objects and uploads it keyed by sha.
|
||||||
|
async function writeSnapshotManifest(
|
||||||
const init: RequestInit & {duplex: 'half'} = {
|
settings: IGitSourceSettings
|
||||||
method: 'PUT',
|
): Promise<void> {
|
||||||
headers: {
|
const sha = settings.commit
|
||||||
'content-length': String(size),
|
const manifest = {
|
||||||
'content-type': 'application/x-tar'
|
repo_key: process.env['GITHUB_REPOSITORY_ID'] as string,
|
||||||
},
|
sha,
|
||||||
body: Readable.toWeb(
|
git_dir: path.join(settings.repositoryPath, '.git')
|
||||||
fs.createReadStream(tmpTar)
|
}
|
||||||
) as unknown as ReadableStream,
|
const dir = manifestDir()
|
||||||
duplex: 'half',
|
await fs.promises.mkdir(dir, {recursive: true})
|
||||||
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
await fs.promises.writeFile(
|
||||||
}
|
path.join(dir, `${sha}.json`),
|
||||||
const res = await fetch(upload.url, init)
|
JSON.stringify(manifest)
|
||||||
if (!res.ok) {
|
)
|
||||||
throw new Error(`snapshot upload answered ${res.status}`)
|
core.info(
|
||||||
}
|
`Cache miss recorded for ${sha}; the WarpBuild agent will upload the snapshot after the job`
|
||||||
await fs.promises.writeFile(
|
)
|
||||||
path.join(gitDir, CACHE_STATE_FILE),
|
|
||||||
`uploaded ${sha}\n`
|
|
||||||
)
|
|
||||||
core.info(
|
|
||||||
`Snapshot uploaded (${size} bytes); jobs checking out ${sha} will skip the GitHub fetch`
|
|
||||||
)
|
|
||||||
} finally {
|
|
||||||
await fs.promises.rm(tmpTar, {force: true})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue