mirror of
https://github.com/actions/checkout.git
synced 2026-07-18 12:00:05 +08:00
remove lfs skip
This commit is contained in:
parent
7a2ed3a023
commit
607bd14933
4 changed files with 289 additions and 42 deletions
|
|
@ -106,24 +106,34 @@ describe('warpbuild mirror cache', () => {
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('serves any fetch-depth and fetch-tags (mirror is full history)', () => {
|
it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => {
|
||||||
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull()
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull()
|
||||||
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toBeNull()
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull()
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain(
|
||||||
|
'explicit shallow depth'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain(
|
||||||
|
'explicit shallow depth'
|
||||||
|
)
|
||||||
expect(
|
expect(
|
||||||
getMirrorCacheSkipReason(settingsFor({fetchTags: true}))
|
getMirrorCacheSkipReason(settingsFor({fetchTags: true}))
|
||||||
).toBeNull()
|
).toBeNull()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips filters, sparse and lfs checkouts', () => {
|
it('skips filters and sparse checkouts', () => {
|
||||||
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
|
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
|
||||||
'a fetch filter is configured'
|
'a fetch filter is configured'
|
||||||
)
|
)
|
||||||
expect(
|
expect(
|
||||||
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
|
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
|
||||||
).toBe('sparse checkout is configured')
|
).toBe('sparse checkout is configured')
|
||||||
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBe(
|
})
|
||||||
'lfs is enabled (lfs objects are not in the mirror)'
|
|
||||||
)
|
it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull()
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1}))
|
||||||
|
).toBeNull()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
145
dist/index.js
vendored
145
dist/index.js
vendored
|
|
@ -41786,6 +41786,7 @@ async function requestBranchUpload(repoKey, ref, sha) {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
|
// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
|
||||||
// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
|
// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
|
||||||
// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
|
// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
|
||||||
|
|
@ -41801,8 +41802,13 @@ const BASE_REFNS = 'refs/wb/base';
|
||||||
const BRANCH_REFNS = 'refs/wb/branch';
|
const BRANCH_REFNS = 'refs/wb/branch';
|
||||||
const UPLOAD_TIP_REF = 'refs/wb/tip';
|
const UPLOAD_TIP_REF = 'refs/wb/tip';
|
||||||
let plan = { mode: 'off', repoKey: '', refKey: '' };
|
let plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
|
// Null = attempt the cache; else a reason to log. We engage only where the result matches
|
||||||
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
|
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
|
||||||
|
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
|
||||||
|
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
|
||||||
|
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
|
||||||
|
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
|
||||||
|
// exactly as upstream — the mirror only accelerates the git-object half.
|
||||||
function getMirrorCacheSkipReason(settings) {
|
function getMirrorCacheSkipReason(settings) {
|
||||||
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
||||||
!process.env['WARPBUILD_HOST_URL']) {
|
!process.env['WARPBUILD_HOST_URL']) {
|
||||||
|
|
@ -41822,15 +41828,18 @@ function getMirrorCacheSkipReason(settings) {
|
||||||
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
return 'no exact commit sha to key on';
|
return 'no exact commit sha to key on';
|
||||||
}
|
}
|
||||||
|
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
|
||||||
|
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
|
||||||
|
if (settings.fetchDepth > 1) {
|
||||||
|
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`;
|
||||||
|
}
|
||||||
if (settings.filter) {
|
if (settings.filter) {
|
||||||
return 'a fetch filter is configured';
|
return 'a fetch filter is configured';
|
||||||
}
|
}
|
||||||
if (settings.sparseCheckout) {
|
if (settings.sparseCheckout) {
|
||||||
return 'sparse checkout is configured';
|
return 'sparse checkout is configured';
|
||||||
}
|
}
|
||||||
if (settings.lfs) {
|
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
|
||||||
return 'lfs is enabled (lfs objects are not in the mirror)';
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
// The durable branch to key the per-branch bundle on. For pull_request events the merge
|
// The durable branch to key the per-branch bundle on. For pull_request events the merge
|
||||||
|
|
@ -41858,6 +41867,14 @@ async function setup(settings) {
|
||||||
setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false');
|
setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false');
|
||||||
return mode;
|
return mode;
|
||||||
}
|
}
|
||||||
|
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
|
||||||
|
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
|
||||||
|
// false since we're no longer serving from cache.
|
||||||
|
async function abandon(settings) {
|
||||||
|
plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
|
setOutput('cache-hit', 'false');
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
async function setupImpl(settings) {
|
async function setupImpl(settings) {
|
||||||
plan = { mode: 'off', repoKey: '', refKey: '' };
|
plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
const skipReason = getMirrorCacheSkipReason(settings);
|
const skipReason = getMirrorCacheSkipReason(settings);
|
||||||
|
|
@ -41871,6 +41888,9 @@ async function setupImpl(settings) {
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`);
|
warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`);
|
||||||
|
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
|
||||||
|
// clean repo (objects just dangle harmlessly).
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
return 'off';
|
return 'off';
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
|
|
@ -41902,6 +41922,14 @@ async function setupInner(settings) {
|
||||||
}
|
}
|
||||||
// Restore: seed the base, then this branch's delta if present.
|
// Restore: seed the base, then this branch's delta if present.
|
||||||
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS);
|
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS);
|
||||||
|
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
|
||||||
|
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
|
||||||
|
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
|
||||||
|
// (depth 1) populates only the target ref, so there we keep the base internal and let the
|
||||||
|
// delta fetch provide just that ref.
|
||||||
|
if (settings.fetchDepth <= 0) {
|
||||||
|
await exposeBaseRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
if (lookup.branch) {
|
if (lookup.branch) {
|
||||||
try {
|
try {
|
||||||
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS);
|
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS);
|
||||||
|
|
@ -41914,20 +41942,42 @@ async function setupInner(settings) {
|
||||||
plan = { mode: 'seeded', repoKey, refKey };
|
plan = { mode: 'seeded', repoKey, refKey };
|
||||||
return 'seeded';
|
return 'seeded';
|
||||||
}
|
}
|
||||||
// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
|
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
|
||||||
// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
|
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
|
||||||
|
// try/catch for normal errors, plus scoped process guards (installed only for the upload
|
||||||
|
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
|
||||||
|
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
|
||||||
|
// exit(0). Safe because nothing checkout-relevant runs after this point.
|
||||||
async function contribute(settings) {
|
async function contribute(settings) {
|
||||||
|
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const guard = (error) => {
|
||||||
|
warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`);
|
||||||
|
process.exit(0);
|
||||||
|
};
|
||||||
|
process.on('uncaughtException', guard);
|
||||||
|
process.on('unhandledRejection', guard);
|
||||||
try {
|
try {
|
||||||
if (plan.mode === 'cold-build') {
|
if (plan.mode === 'cold-build') {
|
||||||
await uploadBaseMirror(settings);
|
await uploadBaseMirror(settings);
|
||||||
}
|
}
|
||||||
else if (plan.mode === 'seeded') {
|
else {
|
||||||
await uploadBranchDelta(settings);
|
await uploadBranchDelta(settings);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (error) {
|
catch (error) {
|
||||||
warning(`WarpBuild mirror upload skipped: ${error}`);
|
warning(`WarpBuild mirror upload skipped: ${error}`);
|
||||||
}
|
}
|
||||||
|
finally {
|
||||||
|
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
|
||||||
|
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
|
||||||
|
if (plan.mode === 'seeded') {
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
|
process.removeListener('uncaughtException', guard);
|
||||||
|
process.removeListener('unhandledRejection', guard);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
|
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
|
||||||
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
|
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
|
||||||
|
|
@ -41950,6 +42000,44 @@ async function seedBundle(repoPath, url, refNs) {
|
||||||
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
|
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Copy the seeded base's branches + tags out to the namespaces upstream uses
|
||||||
|
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
|
||||||
|
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
|
||||||
|
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
|
||||||
|
async function exposeBaseRefs(repoPath) {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'for-each-ref',
|
||||||
|
'--format=create refs/%(refname:lstrip=3) %(objectname)',
|
||||||
|
`${BASE_REFNS}/remotes/origin`,
|
||||||
|
`${BASE_REFNS}/tags`
|
||||||
|
], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
if (!out.trim()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
|
||||||
|
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
|
||||||
|
async function cleanupWbRefs(repoPath) {
|
||||||
|
try {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
if (!out.trim()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`WarpBuild ref cleanup skipped: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
// Cold-build: after the forced full fetch the repo holds all branches under
|
// Cold-build: after the forced full fetch the repo holds all branches under
|
||||||
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
||||||
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
||||||
|
|
@ -42078,7 +42166,9 @@ async function httpPut(url, file) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
function tempBundlePath(tag) {
|
function tempBundlePath(tag) {
|
||||||
return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`);
|
// Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a
|
||||||
|
// shared tmpdir.
|
||||||
|
return external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`);
|
||||||
}
|
}
|
||||||
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
||||||
async function resetGitObjects(gitDir) {
|
async function resetGitObjects(gitDir) {
|
||||||
|
|
@ -42224,15 +42314,28 @@ async function getSource(settings) {
|
||||||
else if (settings.sparseCheckout) {
|
else if (settings.sparseCheckout) {
|
||||||
fetchOptions.filter = 'blob:none';
|
fetchOptions.filter = 'blob:none';
|
||||||
}
|
}
|
||||||
|
let warpbuildFetchDone = false;
|
||||||
if (warpbuildMode === 'seeded') {
|
if (warpbuildMode === 'seeded') {
|
||||||
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
||||||
// objects. No depth — the base is full history, so this stays non-shallow.
|
// objects. No depth — the base is full history, so this stays non-shallow. If it
|
||||||
const refSpec = getRefSpec(settings.ref, settings.commit);
|
// fails for any reason, disengage cleanly and fall through to the standard fetch.
|
||||||
await git.fetch(refSpec, fetchOptions);
|
try {
|
||||||
if (!(await testRef(git, settings.ref, settings.commit))) {
|
const refSpec = getRefSpec(settings.ref, settings.commit);
|
||||||
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
await git.fetch(refSpec, fetchOptions);
|
||||||
`The ref may have been updated after the workflow was triggered.`);
|
if (!(await testRef(git, settings.ref, settings.commit))) {
|
||||||
|
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
||||||
|
`The ref may have been updated after the workflow was triggered.`);
|
||||||
|
}
|
||||||
|
warpbuildFetchDone = true;
|
||||||
}
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`);
|
||||||
|
await abandon(settings);
|
||||||
|
warpbuildMode = 'off';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (warpbuildFetchDone) {
|
||||||
|
// The seeded delta fetch already brought in the target commit.
|
||||||
}
|
}
|
||||||
else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
|
|
@ -42300,8 +42403,6 @@ async function getSource(settings) {
|
||||||
startGroup('Checking out the ref');
|
startGroup('Checking out the ref');
|
||||||
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
|
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
|
||||||
endGroup();
|
endGroup();
|
||||||
// WarpBuild snapshot cache: upload the fetch result on a miss
|
|
||||||
await contribute(settings);
|
|
||||||
// Submodules
|
// Submodules
|
||||||
if (settings.submodules) {
|
if (settings.submodules) {
|
||||||
// Temporarily override global config
|
// Temporarily override global config
|
||||||
|
|
@ -42328,6 +42429,16 @@ async function getSource(settings) {
|
||||||
setOutput('commit', commitSHA.trim());
|
setOutput('commit', commitSHA.trim());
|
||||||
// Check for incorrect pull request merge commit
|
// Check for incorrect pull request merge commit
|
||||||
await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl);
|
await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl);
|
||||||
|
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
|
||||||
|
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
|
||||||
|
// scoped process guard it installs (which ignores any uncaught upload error now that
|
||||||
|
// the checkout is already complete).
|
||||||
|
try {
|
||||||
|
await contribute(settings);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror upload skipped: ${error}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
// Remove auth
|
// Remove auth
|
||||||
|
|
|
||||||
|
|
@ -190,17 +190,32 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
fetchOptions.filter = 'blob:none'
|
fetchOptions.filter = 'blob:none'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let warpbuildFetchDone = false
|
||||||
if (warpbuildMode === 'seeded') {
|
if (warpbuildMode === 'seeded') {
|
||||||
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
||||||
// objects. No depth — the base is full history, so this stays non-shallow.
|
// objects. No depth — the base is full history, so this stays non-shallow. If it
|
||||||
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
|
// fails for any reason, disengage cleanly and fall through to the standard fetch.
|
||||||
await git.fetch(refSpec, fetchOptions)
|
try {
|
||||||
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
|
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
|
||||||
throw new Error(
|
await git.fetch(refSpec, fetchOptions)
|
||||||
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
|
||||||
`The ref may have been updated after the workflow was triggered.`
|
throw new Error(
|
||||||
|
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
||||||
|
`The ref may have been updated after the workflow was triggered.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
warpbuildFetchDone = true
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`
|
||||||
)
|
)
|
||||||
|
await warpbuildMirror.abandon(settings)
|
||||||
|
warpbuildMode = 'off'
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (warpbuildFetchDone) {
|
||||||
|
// The seeded delta fetch already brought in the target commit.
|
||||||
} else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
} else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
let refSpec = refHelper.getRefSpecForAllHistory(
|
let refSpec = refHelper.getRefSpecForAllHistory(
|
||||||
|
|
@ -287,9 +302,6 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
|
await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
// WarpBuild snapshot cache: upload the fetch result on a miss
|
|
||||||
await warpbuildMirror.contribute(settings)
|
|
||||||
|
|
||||||
// Submodules
|
// Submodules
|
||||||
if (settings.submodules) {
|
if (settings.submodules) {
|
||||||
// Temporarily override global config
|
// Temporarily override global config
|
||||||
|
|
@ -332,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
settings.commit,
|
settings.commit,
|
||||||
settings.githubServerUrl
|
settings.githubServerUrl
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
|
||||||
|
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
|
||||||
|
// scoped process guard it installs (which ignores any uncaught upload error now that
|
||||||
|
// the checkout is already complete).
|
||||||
|
try {
|
||||||
|
await warpbuildMirror.contribute(settings)
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(`WarpBuild mirror upload skipped: ${error}`)
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
// Remove auth
|
// Remove auth
|
||||||
if (authHelper) {
|
if (authHelper) {
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
import * as core from '@actions/core'
|
import * as core from '@actions/core'
|
||||||
import * as exec from '@actions/exec'
|
import * as exec from '@actions/exec'
|
||||||
import * as io from '@actions/io'
|
import * as io from '@actions/io'
|
||||||
|
import * as crypto from 'crypto'
|
||||||
import * as fs from 'fs'
|
import * as fs from 'fs'
|
||||||
import * as os from 'os'
|
import * as os from 'os'
|
||||||
import * as path from 'path'
|
import * as path from 'path'
|
||||||
|
|
@ -40,8 +41,13 @@ interface Plan {
|
||||||
}
|
}
|
||||||
let plan: Plan = {mode: 'off', repoKey: '', refKey: ''}
|
let plan: Plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
|
|
||||||
// Null = attempt the cache; else a reason to log. The mirror serves full history for any
|
// Null = attempt the cache; else a reason to log. We engage only where the result matches
|
||||||
// depth, so fetch-depth is not gated; sparse/lfs/filter change the object set we model.
|
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
|
||||||
|
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
|
||||||
|
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
|
||||||
|
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
|
||||||
|
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
|
||||||
|
// exactly as upstream — the mirror only accelerates the git-object half.
|
||||||
export function getMirrorCacheSkipReason(
|
export function getMirrorCacheSkipReason(
|
||||||
settings: IGitSourceSettings
|
settings: IGitSourceSettings
|
||||||
): string | null {
|
): string | null {
|
||||||
|
|
@ -68,15 +74,18 @@ export function getMirrorCacheSkipReason(
|
||||||
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
return 'no exact commit sha to key on'
|
return 'no exact commit sha to key on'
|
||||||
}
|
}
|
||||||
|
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
|
||||||
|
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
|
||||||
|
if (settings.fetchDepth > 1) {
|
||||||
|
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`
|
||||||
|
}
|
||||||
if (settings.filter) {
|
if (settings.filter) {
|
||||||
return 'a fetch filter is configured'
|
return 'a fetch filter is configured'
|
||||||
}
|
}
|
||||||
if (settings.sparseCheckout) {
|
if (settings.sparseCheckout) {
|
||||||
return 'sparse checkout is configured'
|
return 'sparse checkout is configured'
|
||||||
}
|
}
|
||||||
if (settings.lfs) {
|
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
|
||||||
return 'lfs is enabled (lfs objects are not in the mirror)'
|
|
||||||
}
|
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -107,6 +116,15 @@ export async function setup(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
return mode
|
return mode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
|
||||||
|
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
|
||||||
|
// false since we're no longer serving from cache.
|
||||||
|
export async function abandon(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
|
core.setOutput('cache-hit', 'false')
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
|
|
||||||
async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
|
async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
plan = {mode: 'off', repoKey: '', refKey: ''}
|
plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
const skipReason = getMirrorCacheSkipReason(settings)
|
const skipReason = getMirrorCacheSkipReason(settings)
|
||||||
|
|
@ -121,6 +139,9 @@ async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
core.warning(
|
core.warning(
|
||||||
`WarpBuild mirror cache unavailable, using standard checkout: ${error}`
|
`WarpBuild mirror cache unavailable, using standard checkout: ${error}`
|
||||||
)
|
)
|
||||||
|
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
|
||||||
|
// clean repo (objects just dangle harmlessly).
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
return 'off'
|
return 'off'
|
||||||
} finally {
|
} finally {
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
@ -159,6 +180,14 @@ async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
|
|
||||||
// Restore: seed the base, then this branch's delta if present.
|
// Restore: seed the base, then this branch's delta if present.
|
||||||
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS)
|
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS)
|
||||||
|
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
|
||||||
|
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
|
||||||
|
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
|
||||||
|
// (depth 1) populates only the target ref, so there we keep the base internal and let the
|
||||||
|
// delta fetch provide just that ref.
|
||||||
|
if (settings.fetchDepth <= 0) {
|
||||||
|
await exposeBaseRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
if (lookup.branch) {
|
if (lookup.branch) {
|
||||||
try {
|
try {
|
||||||
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS)
|
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS)
|
||||||
|
|
@ -175,17 +204,40 @@ async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
return 'seeded'
|
return 'seeded'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs after checkout, in the same step (`.git` still pristine). Uploads the base (cold)
|
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
|
||||||
// or this branch's refreshed delta (seeded). Best-effort — never fails the checkout.
|
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
|
||||||
|
// try/catch for normal errors, plus scoped process guards (installed only for the upload
|
||||||
|
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
|
||||||
|
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
|
||||||
|
// exit(0). Safe because nothing checkout-relevant runs after this point.
|
||||||
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const guard = (error: unknown): void => {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild mirror upload error ignored — checkout already complete: ${error}`
|
||||||
|
)
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
process.on('uncaughtException', guard)
|
||||||
|
process.on('unhandledRejection', guard)
|
||||||
try {
|
try {
|
||||||
if (plan.mode === 'cold-build') {
|
if (plan.mode === 'cold-build') {
|
||||||
await uploadBaseMirror(settings)
|
await uploadBaseMirror(settings)
|
||||||
} else if (plan.mode === 'seeded') {
|
} else {
|
||||||
await uploadBranchDelta(settings)
|
await uploadBranchDelta(settings)
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.warning(`WarpBuild mirror upload skipped: ${error}`)
|
core.warning(`WarpBuild mirror upload skipped: ${error}`)
|
||||||
|
} finally {
|
||||||
|
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
|
||||||
|
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
|
||||||
|
if (plan.mode === 'seeded') {
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
|
process.removeListener('uncaughtException', guard)
|
||||||
|
process.removeListener('unhandledRejection', guard)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -214,6 +266,53 @@ async function seedBundle(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy the seeded base's branches + tags out to the namespaces upstream uses
|
||||||
|
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
|
||||||
|
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
|
||||||
|
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
|
||||||
|
async function exposeBaseRefs(repoPath: string): Promise<void> {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
[
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'for-each-ref',
|
||||||
|
'--format=create refs/%(refname:lstrip=3) %(objectname)',
|
||||||
|
`${BASE_REFNS}/remotes/origin`,
|
||||||
|
`${BASE_REFNS}/tags`
|
||||||
|
],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
if (!out.trim()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
|
||||||
|
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
|
||||||
|
async function cleanupWbRefs(repoPath: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
if (!out.trim()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`WarpBuild ref cleanup skipped: ${error}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Cold-build: after the forced full fetch the repo holds all branches under
|
// Cold-build: after the forced full fetch the repo holds all branches under
|
||||||
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
||||||
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
||||||
|
|
@ -363,7 +462,12 @@ async function httpPut(url: string, file: string): Promise<void> {
|
||||||
}
|
}
|
||||||
|
|
||||||
function tempBundlePath(tag: string): string {
|
function tempBundlePath(tag: string): string {
|
||||||
return path.join(os.tmpdir(), `wb-${tag}-${process.pid}-${Date.now()}.bundle`)
|
// Unpredictable name so a bundle write can't be pre-empted by a symlink planted on a
|
||||||
|
// shared tmpdir.
|
||||||
|
return path.join(
|
||||||
|
os.tmpdir(),
|
||||||
|
`wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue