diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts index e6250e9..1d860f4 100644 --- a/__test__/warpbuild-mirror.test.ts +++ b/__test__/warpbuild-mirror.test.ts @@ -106,24 +106,34 @@ describe('warpbuild mirror cache', () => { ) }) - it('serves any fetch-depth and fetch-tags (mirror is full history)', () => { + it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => { expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull() - expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain( + 'explicit shallow depth' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain( + 'explicit shallow depth' + ) expect( getMirrorCacheSkipReason(settingsFor({fetchTags: true})) ).toBeNull() }) - it('skips filters, sparse and lfs checkouts', () => { + it('skips filters and sparse checkouts', () => { expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( 'a fetch filter is configured' ) expect( getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) ).toBe('sparse checkout is configured') - expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe( - 'lfs is enabled (lfs objects are not in the mirror)' - ) + }) + + it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => { + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull() + expect( + getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1})) + ).toBeNull() }) }) diff --git a/dist/index.js b/dist/index.js index 04eb13a..943b39f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -41786,6 +41786,7 @@ async function requestBranchUpload(repoKey, ref, sha) { + // WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on // S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is // only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, @@ -41801,8 +41802,13 @@ const BASE_REFNS = 'refs/wb/base'; const BRANCH_REFNS = 'refs/wb/branch'; const UPLOAD_TIP_REF = 'refs/wb/tip'; let plan = { mode: 'off', repoKey: '', refKey: '' }; -// Null = attempt the cache; else a reason to log. The mirror serves full history for any -// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. function getMirrorCacheSkipReason(settings) { if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || !process.env['WARPBUILD_HOST_URL']) { @@ -41822,15 +41828,18 @@ function getMirrorCacheSkipReason(settings) { if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on'; } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`; + } if (settings.filter) { return 'a fetch filter is configured'; } if (settings.sparseCheckout) { return 'sparse checkout is configured'; } - if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the mirror)'; - } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. return null; } // The durable branch to key the per-branch bundle on. For pull_request events the merge @@ -41858,6 +41867,14 @@ async function setup(settings) { setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false'); return mode; } +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +async function abandon(settings) { + plan = { mode: 'off', repoKey: '', refKey: '' }; + setOutput('cache-hit', 'false'); + await cleanupWbRefs(settings.repositoryPath); +} async function setupImpl(settings) { plan = { mode: 'off', repoKey: '', refKey: '' }; const skipReason = getMirrorCacheSkipReason(settings); @@ -41871,6 +41888,9 @@ async function setupImpl(settings) { } catch (error) { warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath); return 'off'; } finally { @@ -41902,6 +41922,14 @@ async function setupInner(settings) { } // Restore: seed the base, then this branch's delta if present. await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS); + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath); + } if (lookup.branch) { try { await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS); @@ -41914,20 +41942,42 @@ async function setupInner(settings) { plan = { mode: 'seeded', repoKey, refKey }; return 'seeded'; } -// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) -// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. async function contribute(settings) { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return; + } + const guard = (error) => { + warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`); + process.exit(0); + }; + process.on('uncaughtException', guard); + process.on('unhandledRejection', guard); try { if (plan.mode === 'cold-build') { await uploadBaseMirror(settings); } - else if (plan.mode === 'seeded') { + else { await uploadBranchDelta(settings); } } catch (error) { warning(`WarpBuild mirror upload skipped: ${error}`); } + finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath); + } + process.removeListener('uncaughtException', guard); + process.removeListener('unhandledRejection', guard); + } } // Download a bundle and fetch its refs into refNs/* so its objects seed the local repo // (and count as "haves" for the delta negotiation). The bundle is a git-validated file; @@ -41950,6 +42000,44 @@ async function seedBundle(repoPath, url, refNs) { await external_fs_namespaceObject.promises.rm(tmp, { force: true }); } } +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath) { + let out = ''; + await exec_exec('git', [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); +} +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath) { + try { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); + } + catch (error) { + core_debug(`WarpBuild ref cleanup skipped: ${error}`); + } +} // Cold-build: after the forced full fetch the repo holds all branches under // refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would // also sweep up the ephemeral triggering PR merge ref) as the base and upload. @@ -42078,7 +42166,9 @@ async function httpPut(url, file) { } } function tempBundlePath(tag) { - return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`); + // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a + // shared tmpdir. + return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`); } // Kept for callers/tests: reset .git/objects to the empty `git init` shape. async function resetGitObjects(gitDir) { @@ -42224,15 +42314,28 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } + let warpbuildFetchDone = false; if (warpbuildMode === 'seeded') { // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded - // objects. No depth — the base is full history, so this stays non-shallow. - const refSpec = getRefSpec(settings.ref, settings.commit); - await git.fetch(refSpec, fetchOptions); - if (!(await testRef(git, settings.ref, settings.commit))) { - throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + - `The ref may have been updated after the workflow was triggered.`); + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = getRefSpec(settings.ref, settings.commit); + await git.fetch(refSpec, fetchOptions); + if (!(await testRef(git, settings.ref, settings.commit))) { + throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.`); + } + warpbuildFetchDone = true; } + catch (error) { + warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`); + await abandon(settings); + warpbuildMode = 'off'; + } + } + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags @@ -42300,8 +42403,6 @@ async function getSource(settings) { startGroup('Checking out the ref'); await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); endGroup(); - // WarpBuild snapshot cache: upload the fetch result on a miss - await contribute(settings); // Submodules if (settings.submodules) { // Temporarily override global config @@ -42328,6 +42429,16 @@ async function getSource(settings) { setOutput('commit', commitSHA.trim()); // Check for incorrect pull request merge commit await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl); + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await contribute(settings); + } + catch (error) { + warning(`WarpBuild mirror upload skipped: ${error}`); + } } finally { // Remove auth diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index fbdf9d7..df892aa 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -190,17 +190,32 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } + let warpbuildFetchDone = false if (warpbuildMode === 'seeded') { // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded - // objects. No depth — the base is full history, so this stays non-shallow. - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) - if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { - throw new Error( - `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + - `The ref may have been updated after the workflow was triggered.` + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + throw new Error( + `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.` + ) + } + warpbuildFetchDone = true + } catch (error) { + core.warning( + `WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}` ) + await warpbuildMirror.abandon(settings) + warpbuildMode = 'off' } + } + + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( @@ -287,9 +302,6 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) core.endGroup() - // WarpBuild snapshot cache: upload the fetch result on a miss - await warpbuildMirror.contribute(settings) - // Submodules if (settings.submodules) { // Temporarily override global config @@ -332,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise { settings.commit, settings.githubServerUrl ) + + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await warpbuildMirror.contribute(settings) + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } } finally { // Remove auth if (authHelper) { diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts index 42d85c0..3ed93ad 100644 --- a/src/warpbuild/mirror-cache.ts +++ b/src/warpbuild/mirror-cache.ts @@ -2,6 +2,7 @@ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as io from '@actions/io' +import * as crypto from 'crypto' import * as fs from 'fs' import * as os from 'os' import * as path from 'path' @@ -40,8 +41,13 @@ interface Plan { } let plan: Plan = {mode: 'off', repoKey: '', refKey: ''} -// Null = attempt the cache; else a reason to log. The mirror serves full history for any -// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model. +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. export function getMirrorCacheSkipReason( settings: IGitSourceSettings ): string | null { @@ -68,15 +74,18 @@ export function getMirrorCacheSkipReason( if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { return 'no exact commit sha to key on' } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout` + } if (settings.filter) { return 'a fetch filter is configured' } if (settings.sparseCheckout) { return 'sparse checkout is configured' } - if (settings.lfs) { - return 'lfs is enabled (lfs objects are not in the mirror)' - } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. return null } @@ -107,6 +116,15 @@ export async function setup(settings: IGitSourceSettings): Promise { return mode } +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +export async function abandon(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} + core.setOutput('cache-hit', 'false') + await cleanupWbRefs(settings.repositoryPath) +} + async function setupImpl(settings: IGitSourceSettings): Promise { plan = {mode: 'off', repoKey: '', refKey: ''} const skipReason = getMirrorCacheSkipReason(settings) @@ -121,6 +139,9 @@ async function setupImpl(settings: IGitSourceSettings): Promise { core.warning( `WarpBuild mirror cache unavailable, using standard checkout: ${error}` ) + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath) return 'off' } finally { core.endGroup() @@ -159,6 +180,14 @@ async function setupInner(settings: IGitSourceSettings): Promise { // Restore: seed the base, then this branch's delta if present. await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS) + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath) + } if (lookup.branch) { try { await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS) @@ -175,17 +204,40 @@ async function setupInner(settings: IGitSourceSettings): Promise { return 'seeded' } -// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold) -// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout. +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. export async function contribute(settings: IGitSourceSettings): Promise { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return + } + const guard = (error: unknown): void => { + core.warning( + `WarpBuild mirror upload error ignored — checkout already complete: ${error}` + ) + process.exit(0) + } + process.on('uncaughtException', guard) + process.on('unhandledRejection', guard) try { if (plan.mode === 'cold-build') { await uploadBaseMirror(settings) - } else if (plan.mode === 'seeded') { + } else { await uploadBranchDelta(settings) } } catch (error) { core.warning(`WarpBuild mirror upload skipped: ${error}`) + } finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath) + } + process.removeListener('uncaughtException', guard) + process.removeListener('unhandledRejection', guard) } } @@ -214,6 +266,53 @@ async function seedBundle( } } +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) +} + +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath: string): Promise { + try { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) + } catch (error) { + core.debug(`WarpBuild ref cleanup skipped: ${error}`) + } +} + // Cold-build: after the forced full fetch the repo holds all branches under // refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would // also sweep up the ephemeral triggering PR merge ref) as the base and upload. @@ -363,7 +462,12 @@ async function httpPut(url: string, file: string): Promise { } function tempBundlePath(tag: string): string { - return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`) + // Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a + // shared tmpdir. + return path.join( + os.tmpdir(), + `wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle` + ) } // Kept for callers/tests: reset .git/objects to the empty `git init` shape.