mirror/checkout
1
0
Fork 0
mirror of https://github.com/actions/checkout.git synced 2026-07-18 03:50:07 +08:00
Code Issues Projects Releases Packages Wiki Activity

remove lfs skip

Browse source
This commit is contained in:
darshanime 2026-07-13 17:19:38 +05:30
parent 7a2ed3a023
commit 607bd14933
4 changed files with 289 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

22
__test__/warpbuild-mirror.test.ts
View file

@ -106,24 +106,34 @@ describe('warpbuild mirror cache', () => {
)
})
it('serves any fetch-depth and fetch-tags (mirror is full history)', () => {
it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => {
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull()
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull()
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull()
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain(
'explicit shallow depth'
)
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain(
'explicit shallow depth'
)
expect(
getMirrorCacheSkipReason(settingsFor({fetchTags: true}))
).toBeNull()
})
it('skips filters, sparse and lfs checkouts', () => {
it('skips filters and sparse checkouts', () => {
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
'a fetch filter is configured'
)
expect(
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
).toBe('sparse checkout is configured')
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe(
'lfs is enabled (lfs objects are not in the mirror)'
)
})
it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => {
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull()
expect(
getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1}))
).toBeNull()
})
})

145
dist/index.js vendored
View file

@ -41786,6 +41786,7 @@ async function requestBranchUpload(repoKey, ref, sha) {
// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
@ -41801,8 +41802,13 @@ const BASE_REFNS = 'refs/wb/base';
const BRANCH_REFNS = 'refs/wb/branch';
const UPLOAD_TIP_REF = 'refs/wb/tip';
let plan = { mode: 'off', repoKey: '', refKey: '' };
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
// Null = attempt the cache; else a reason to log. We engage only where the result matches
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
// exactly as upstream — the mirror only accelerates the git-object half.
function getMirrorCacheSkipReason(settings) {
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
!process.env['WARPBUILD_HOST_URL']) {
@ -41822,15 +41828,18 @@ function getMirrorCacheSkipReason(settings) {
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on';
}
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
if (settings.fetchDepth > 1) {
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`;
}
if (settings.filter) {
return 'a fetch filter is configured';
}
if (settings.sparseCheckout) {
return 'sparse checkout is configured';
}
if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the mirror)';
}
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
return null;
}
// The durable branch to key the per-branch bundle on. For pull_request events the merge
@ -41858,6 +41867,14 @@ async function setup(settings) {
setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false');
return mode;
}
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
// false since we're no longer serving from cache.
async function abandon(settings) {
plan = { mode: 'off', repoKey: '', refKey: '' };
setOutput('cache-hit', 'false');
await cleanupWbRefs(settings.repositoryPath);
}
async function setupImpl(settings) {
plan = { mode: 'off', repoKey: '', refKey: '' };
const skipReason = getMirrorCacheSkipReason(settings);
@ -41871,6 +41888,9 @@ async function setupImpl(settings) {
}
catch (error) {
warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`);
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
// clean repo (objects just dangle harmlessly).
await cleanupWbRefs(settings.repositoryPath);
return 'off';
}
finally {
@ -41902,6 +41922,14 @@ async function setupInner(settings) {
}
// Restore: seed the base, then this branch's delta if present.
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS);
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
// (depth 1) populates only the target ref, so there we keep the base internal and let the
// delta fetch provide just that ref.
if (settings.fetchDepth <= 0) {
await exposeBaseRefs(settings.repositoryPath);
}
if (lookup.branch) {
try {
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS);
@ -41914,20 +41942,42 @@ async function setupInner(settings) {
plan = { mode: 'seeded', repoKey, refKey };
return 'seeded';
}
// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
// try/catch for normal errors, plus scoped process guards (installed only for the upload
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
// exit(0). Safe because nothing checkout-relevant runs after this point.
async function contribute(settings) {
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
return;
}
const guard = (error) => {
warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`);
process.exit(0);
};
process.on('uncaughtException', guard);
process.on('unhandledRejection', guard);
try {
if (plan.mode === 'cold-build') {
await uploadBaseMirror(settings);
}
else if (plan.mode === 'seeded') {
else {
await uploadBranchDelta(settings);
}
}
catch (error) {
warning(`WarpBuild mirror upload skipped: ${error}`);
}
finally {
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
if (plan.mode === 'seeded') {
await cleanupWbRefs(settings.repositoryPath);
}
process.removeListener('uncaughtException', guard);
process.removeListener('unhandledRejection', guard);
}
}
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
@ -41950,6 +42000,44 @@ async function seedBundle(repoPath, url, refNs) {
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
}
}
// Copy the seeded base's branches + tags out to the namespaces upstream uses
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
async function exposeBaseRefs(repoPath) {
let out = '';
await exec_exec('git', [
'-C',
repoPath,
'for-each-ref',
'--format=create refs/%(refname:lstrip=3) %(objectname)',
`${BASE_REFNS}/remotes/origin`,
`${BASE_REFNS}/tags`
], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
if (!out.trim()) {
return;
}
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
input: Buffer.from(out)
});
}
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
async function cleanupWbRefs(repoPath) {
try {
let out = '';
await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
if (!out.trim()) {
return;
}
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
input: Buffer.from(out)
});
}
catch (error) {
core_debug(`WarpBuild ref cleanup skipped: ${error}`);
}
}
// Cold-build: after the forced full fetch the repo holds all branches under
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
@ -42078,7 +42166,9 @@ async function httpPut(url, file) {
}
}
function tempBundlePath(tag) {
return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`);
// Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a
// shared tmpdir.
return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`);
}
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
async function resetGitObjects(gitDir) {
@ -42224,15 +42314,28 @@ async function getSource(settings) {
else if (settings.sparseCheckout) {
fetchOptions.filter = 'blob:none';
}
let warpbuildFetchDone = false;
if (warpbuildMode === 'seeded') {
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
// objects. No depth — the base is full history, so this stays non-shallow.
const refSpec = getRefSpec(settings.ref, settings.commit);
await git.fetch(refSpec, fetchOptions);
if (!(await testRef(git, settings.ref, settings.commit))) {
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`);
// objects. No depth — the base is full history, so this stays non-shallow. If it
// fails for any reason, disengage cleanly and fall through to the standard fetch.
try {
const refSpec = getRefSpec(settings.ref, settings.commit);
await git.fetch(refSpec, fetchOptions);
if (!(await testRef(git, settings.ref, settings.commit))) {
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`);
}
warpbuildFetchDone = true;
}
catch (error) {
warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`);
await abandon(settings);
warpbuildMode = 'off';
}
}
if (warpbuildFetchDone) {
// The seeded delta fetch already brought in the target commit.
}
else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
// Fetch all branches and tags
@ -42300,8 +42403,6 @@ async function getSource(settings) {
startGroup('Checking out the ref');
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
endGroup();
// WarpBuild snapshot cache: upload the fetch result on a miss
await contribute(settings);
// Submodules
if (settings.submodules) {
// Temporarily override global config
@ -42328,6 +42429,16 @@ async function getSource(settings) {
setOutput('commit', commitSHA.trim());
// Check for incorrect pull request merge commit
await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl);
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
// scoped process guard it installs (which ignores any uncaught upload error now that
// the checkout is already complete).
try {
await contribute(settings);
}
catch (error) {
warning(`WarpBuild mirror upload skipped: ${error}`);
}
}
finally {
// Remove auth

42
src/git-source-provider.ts
View file

@ -190,17 +190,32 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
fetchOptions.filter = 'blob:none'
}
let warpbuildFetchDone = false
if (warpbuildMode === 'seeded') {
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
// objects. No depth — the base is full history, so this stays non-shallow.
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
await git.fetch(refSpec, fetchOptions)
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
throw new Error(
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`
// objects. No depth — the base is full history, so this stays non-shallow. If it
// fails for any reason, disengage cleanly and fall through to the standard fetch.
try {
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
await git.fetch(refSpec, fetchOptions)
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
throw new Error(
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
`The ref may have been updated after the workflow was triggered.`
)
}
warpbuildFetchDone = true
} catch (error) {
core.warning(
`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`
)
await warpbuildMirror.abandon(settings)
warpbuildMode = 'off'
}
}
if (warpbuildFetchDone) {
// The seeded delta fetch already brought in the target commit.
} else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
// Fetch all branches and tags
let refSpec = refHelper.getRefSpecForAllHistory(
@ -287,9 +302,6 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
core.endGroup()
// WarpBuild snapshot cache: upload the fetch result on a miss
await warpbuildMirror.contribute(settings)
// Submodules
if (settings.submodules) {
// Temporarily override global config
@ -332,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
settings.commit,
settings.githubServerUrl
)
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
// scoped process guard it installs (which ignores any uncaught upload error now that
// the checkout is already complete).
try {
await warpbuildMirror.contribute(settings)
} catch (error) {
core.warning(`WarpBuild mirror upload skipped: ${error}`)
}
} finally {
// Remove auth
if (authHelper) {

122
src/warpbuild/mirror-cache.ts
View file

@ -2,6 +2,7 @@
import * as core from '@actions/core'
import * as exec from '@actions/exec'
import * as io from '@actions/io'
import * as crypto from 'crypto'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
@ -40,8 +41,13 @@ interface Plan {
}
let plan: Plan = {mode: 'off', repoKey: '', refKey: ''}
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
// Null = attempt the cache; else a reason to log. We engage only where the result matches
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
// exactly as upstream — the mirror only accelerates the git-object half.
export function getMirrorCacheSkipReason(
settings: IGitSourceSettings
): string | null {
@ -68,15 +74,18 @@ export function getMirrorCacheSkipReason(
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
return 'no exact commit sha to key on'
}
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
if (settings.fetchDepth > 1) {
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`
}
if (settings.filter) {
return 'a fetch filter is configured'
}
if (settings.sparseCheckout) {
return 'sparse checkout is configured'
}
if (settings.lfs) {
return 'lfs is enabled (lfs objects are not in the mirror)'
}
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
return null
}
@ -107,6 +116,15 @@ export async function setup(settings: IGitSourceSettings): Promise<MirrorMode> {
return mode
}
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
// false since we're no longer serving from cache.
export async function abandon(settings: IGitSourceSettings): Promise<void> {
plan = {mode: 'off', repoKey: '', refKey: ''}
core.setOutput('cache-hit', 'false')
await cleanupWbRefs(settings.repositoryPath)
}
async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
plan = {mode: 'off', repoKey: '', refKey: ''}
const skipReason = getMirrorCacheSkipReason(settings)
@ -121,6 +139,9 @@ async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
core.warning(
`WarpBuild mirror cache unavailable, using standard checkout: ${error}`
)
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
// clean repo (objects just dangle harmlessly).
await cleanupWbRefs(settings.repositoryPath)
return 'off'
} finally {
core.endGroup()
@ -159,6 +180,14 @@ async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
// Restore: seed the base, then this branch's delta if present.
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS)
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
// (depth 1) populates only the target ref, so there we keep the base internal and let the
// delta fetch provide just that ref.
if (settings.fetchDepth <= 0) {
await exposeBaseRefs(settings.repositoryPath)
}
if (lookup.branch) {
try {
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS)
@ -175,17 +204,40 @@ async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
return 'seeded'
}
// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
// try/catch for normal errors, plus scoped process guards (installed only for the upload
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
// exit(0). Safe because nothing checkout-relevant runs after this point.
export async function contribute(settings: IGitSourceSettings): Promise<void> {
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
return
}
const guard = (error: unknown): void => {
core.warning(
`WarpBuild mirror upload error ignored — checkout already complete: ${error}`
)
process.exit(0)
}
process.on('uncaughtException', guard)
process.on('unhandledRejection', guard)
try {
if (plan.mode === 'cold-build') {
await uploadBaseMirror(settings)
} else if (plan.mode === 'seeded') {
} else {
await uploadBranchDelta(settings)
}
} catch (error) {
core.warning(`WarpBuild mirror upload skipped: ${error}`)
} finally {
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
if (plan.mode === 'seeded') {
await cleanupWbRefs(settings.repositoryPath)
}
process.removeListener('uncaughtException', guard)
process.removeListener('unhandledRejection', guard)
}
}
@ -214,6 +266,53 @@ async function seedBundle(
}
}
// Copy the seeded base's branches + tags out to the namespaces upstream uses
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
async function exposeBaseRefs(repoPath: string): Promise<void> {
let out = ''
await exec.exec(
'git',
[
'-C',
repoPath,
'for-each-ref',
'--format=create refs/%(refname:lstrip=3) %(objectname)',
`${BASE_REFNS}/remotes/origin`,
`${BASE_REFNS}/tags`
],
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
)
if (!out.trim()) {
return
}
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
input: Buffer.from(out)
})
}
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
async function cleanupWbRefs(repoPath: string): Promise<void> {
try {
let out = ''
await exec.exec(
'git',
['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'],
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
)
if (!out.trim()) {
return
}
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
input: Buffer.from(out)
})
} catch (error) {
core.debug(`WarpBuild ref cleanup skipped: ${error}`)
}
}
// Cold-build: after the forced full fetch the repo holds all branches under
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
@ -363,7 +462,12 @@ async function httpPut(url: string, file: string): Promise<void> {
}
function tempBundlePath(tag: string): string {
return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`)
// Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a
// shared tmpdir.
return path.join(
os.tmpdir(),
`wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle`
)
}
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.