mirror of
https://github.com/actions/checkout.git
synced 2026-07-18 03:50:07 +08:00
Darshan Chaudhary
GitHub
commit
33796bf525
8 changed files with 1797 additions and 10 deletions
17
README.md
17
README.md
|
|
@ -1,3 +1,20 @@
|
||||||
|
# WarpBuild Checkout
|
||||||
|
|
||||||
|
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
|
||||||
|
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
|
||||||
|
objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA.
|
||||||
|
A later job checking out the same commit restores that snapshot and skips the fetch from
|
||||||
|
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
|
||||||
|
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
|
||||||
|
|
||||||
|
- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS);
|
||||||
|
everything else runs exactly like upstream.
|
||||||
|
- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners.
|
||||||
|
- Fail-open — any cache error degrades to stock `actions/checkout` behavior.
|
||||||
|
- All fork code lives in `src/warpbuild/`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
[](https://github.com/actions/checkout/actions/workflows/test.yml)
|
[](https://github.com/actions/checkout/actions/workflows/test.yml)
|
||||||
|
|
||||||
# Checkout v7
|
# Checkout v7
|
||||||
|
|
|
||||||
273
__test__/warpbuild-mirror.test.ts
Normal file
273
__test__/warpbuild-mirror.test.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
||||||
|
import {
|
||||||
|
describe,
|
||||||
|
it,
|
||||||
|
expect,
|
||||||
|
beforeEach,
|
||||||
|
afterEach,
|
||||||
|
afterAll
|
||||||
|
} from '@jest/globals'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as os from 'os'
|
||||||
|
import * as path from 'path'
|
||||||
|
import {
|
||||||
|
SKIP_NOT_WARPBUILD,
|
||||||
|
computeRefKey,
|
||||||
|
getMirrorCacheSkipReason,
|
||||||
|
resetGitObjects
|
||||||
|
} from '../src/warpbuild/mirror-cache.js'
|
||||||
|
import {
|
||||||
|
lookupRestore,
|
||||||
|
requestBaseUpload,
|
||||||
|
requestBranchUpload
|
||||||
|
} from '../src/warpbuild/backend-api.js'
|
||||||
|
import {IGitSourceSettings} from '../src/git-source-settings.js'
|
||||||
|
|
||||||
|
const WB_ENV = [
|
||||||
|
'WARPBUILD_RUNNER_VERIFICATION_TOKEN',
|
||||||
|
'WARPBUILD_HOST_URL',
|
||||||
|
'GITHUB_REPOSITORY_ID',
|
||||||
|
'GITHUB_REPOSITORY',
|
||||||
|
'GITHUB_REF',
|
||||||
|
'GITHUB_BASE_REF'
|
||||||
|
]
|
||||||
|
const savedEnv: {[key: string]: string | undefined} = {}
|
||||||
|
for (const key of WB_ENV) {
|
||||||
|
savedEnv[key] = process.env[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7'
|
||||||
|
|
||||||
|
function settingsFor(
|
||||||
|
overrides: Partial<IGitSourceSettings> = {}
|
||||||
|
): IGitSourceSettings {
|
||||||
|
return {
|
||||||
|
repositoryOwner: 'octocat',
|
||||||
|
repositoryName: 'hello-world',
|
||||||
|
repositoryPath: '/tmp/does-not-matter',
|
||||||
|
ref: 'refs/heads/main',
|
||||||
|
commit: SHA,
|
||||||
|
fetchDepth: 1,
|
||||||
|
fetchTags: false,
|
||||||
|
filter: undefined,
|
||||||
|
sparseCheckout: undefined,
|
||||||
|
lfs: false,
|
||||||
|
...overrides
|
||||||
|
} as unknown as IGitSourceSettings
|
||||||
|
}
|
||||||
|
|
||||||
|
function setWarpBuildEnv(): void {
|
||||||
|
process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token'
|
||||||
|
process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev'
|
||||||
|
process.env['GITHUB_REPOSITORY_ID'] = '123456789'
|
||||||
|
process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world'
|
||||||
|
delete process.env['GITHUB_REF']
|
||||||
|
delete process.env['GITHUB_BASE_REF']
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('warpbuild mirror cache', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
setWarpBuildEnv()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
for (const key of WB_ENV) {
|
||||||
|
if (savedEnv[key] === undefined) {
|
||||||
|
delete process.env[key]
|
||||||
|
} else {
|
||||||
|
process.env[key] = savedEnv[key]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('getMirrorCacheSkipReason', () => {
|
||||||
|
it('returns null for the default checkout shape', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor())).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports a non-WarpBuild runner', () => {
|
||||||
|
delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN']
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports repository inputs that are not the workflow repo', () => {
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'}))
|
||||||
|
).toBe(
|
||||||
|
"repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('requires an exact commit sha', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe(
|
||||||
|
'no exact commit sha to key on'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe(
|
||||||
|
'no exact commit sha to key on'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull()
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull()
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain(
|
||||||
|
'explicit shallow depth'
|
||||||
|
)
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain(
|
||||||
|
'explicit shallow depth'
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({fetchTags: true}))
|
||||||
|
).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('skips filters and sparse checkouts', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe(
|
||||||
|
'a fetch filter is configured'
|
||||||
|
)
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']}))
|
||||||
|
).toBe('sparse checkout is configured')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => {
|
||||||
|
expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull()
|
||||||
|
expect(
|
||||||
|
getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1}))
|
||||||
|
).toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('computeRefKey', () => {
|
||||||
|
it('keys on the pushed branch (GITHUB_REF)', () => {
|
||||||
|
process.env['GITHUB_REF'] = 'refs/heads/feature/x'
|
||||||
|
expect(computeRefKey(settingsFor())).toBe('feature/x')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('keys on the base branch for pull requests (merge sha is synthetic)', () => {
|
||||||
|
process.env['GITHUB_REF'] = 'refs/pull/42/merge'
|
||||||
|
process.env['GITHUB_BASE_REF'] = 'main'
|
||||||
|
expect(computeRefKey(settingsFor())).toBe('main')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is empty for tags and bare pull refs (base only, no roll)', () => {
|
||||||
|
process.env['GITHUB_REF'] = 'refs/tags/v1.2.3'
|
||||||
|
expect(computeRefKey(settingsFor())).toBe('')
|
||||||
|
process.env['GITHUB_REF'] = 'refs/pull/42/merge'
|
||||||
|
expect(computeRefKey(settingsFor())).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is empty for a detached sha checkout', () => {
|
||||||
|
expect(computeRefKey(settingsFor({ref: SHA}))).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('passes through an already-short branch name', () => {
|
||||||
|
expect(computeRefKey(settingsFor({ref: 'main'}))).toBe('main')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('resetGitObjects', () => {
|
||||||
|
it('restores the empty git-init object layout, dropping restored junk', async () => {
|
||||||
|
const gitDir = await fs.promises.mkdtemp(
|
||||||
|
path.join(os.tmpdir(), 'wb-reset-')
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
path.join(gitDir, 'objects', 'ab'),
|
||||||
|
'junk loose object'
|
||||||
|
)
|
||||||
|
await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n')
|
||||||
|
|
||||||
|
await resetGitObjects(gitDir)
|
||||||
|
|
||||||
|
expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false)
|
||||||
|
expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false)
|
||||||
|
expect(
|
||||||
|
(await fs.promises.readdir(path.join(gitDir, 'objects'))).sort()
|
||||||
|
).toEqual(['info', 'pack'])
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(gitDir, {recursive: true, force: true})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('backend api http contract', () => {
|
||||||
|
const realFetch = globalThis.fetch
|
||||||
|
let lastUrl = ''
|
||||||
|
let lastInit: RequestInit | undefined
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = realFetch
|
||||||
|
})
|
||||||
|
|
||||||
|
function stubFetch(status: number, body: unknown): void {
|
||||||
|
globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
|
||||||
|
lastUrl = String(input)
|
||||||
|
lastInit = init
|
||||||
|
return new Response(JSON.stringify(body), {status})
|
||||||
|
}) as typeof fetch
|
||||||
|
}
|
||||||
|
|
||||||
|
it('lookupRestore maps 200 to a restore plan keyed by ref', async () => {
|
||||||
|
stubFetch(200, {
|
||||||
|
base: {url: 'https://s3/base', size_bytes: 42},
|
||||||
|
branch: {url: 'https://s3/branch'}
|
||||||
|
})
|
||||||
|
const result = await lookupRestore('123', 'main')
|
||||||
|
expect(result.kind).toBe('restore')
|
||||||
|
if (result.kind === 'restore') {
|
||||||
|
expect(result.base.url).toBe('https://s3/base')
|
||||||
|
expect(result.branch?.url).toBe('https://s3/branch')
|
||||||
|
}
|
||||||
|
expect(lastUrl).toContain('ref=main')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lookupRestore maps a null branch (base but no delta yet)', async () => {
|
||||||
|
stubFetch(200, {base: {url: 'https://s3/base'}, branch: null})
|
||||||
|
const result = await lookupRestore('123', 'main')
|
||||||
|
expect(result.kind).toBe('restore')
|
||||||
|
if (result.kind === 'restore') {
|
||||||
|
expect(result.branch).toBeNull()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lookupRestore maps 404 to cold, 403 to disabled, 500 to error', async () => {
|
||||||
|
stubFetch(404, {sub_code: 'NFE_GITMIRROR'})
|
||||||
|
expect((await lookupRestore('123', 'main')).kind).toBe('cold')
|
||||||
|
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
|
||||||
|
expect((await lookupRestore('123', 'main')).kind).toBe('disabled')
|
||||||
|
stubFetch(500, {})
|
||||||
|
expect((await lookupRestore('123', 'main')).kind).toBe('error')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lookupRestore maps network failure to error', async () => {
|
||||||
|
globalThis.fetch = (async () => {
|
||||||
|
throw new Error('boom')
|
||||||
|
}) as typeof fetch
|
||||||
|
expect((await lookupRestore('123', 'main')).kind).toBe('error')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('requestBaseUpload maps 200 to a grant, 409 to locked', async () => {
|
||||||
|
stubFetch(200, {url: 'https://s3/put-base'})
|
||||||
|
const grant = await requestBaseUpload('123')
|
||||||
|
expect(grant.kind).toBe('grant')
|
||||||
|
if (grant.kind === 'grant') {
|
||||||
|
expect(grant.url).toBe('https://s3/put-base')
|
||||||
|
}
|
||||||
|
expect(lastInit?.method).toBe('POST')
|
||||||
|
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
|
||||||
|
expect((await requestBaseUpload('123')).kind).toBe('locked')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('requestBranchUpload maps 200 to a grant, 403 to disabled', async () => {
|
||||||
|
stubFetch(200, {url: 'https://s3/put-branch'})
|
||||||
|
expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe('grant')
|
||||||
|
stubFetch(403, {})
|
||||||
|
expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe(
|
||||||
|
'disabled'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -112,6 +112,8 @@ outputs:
|
||||||
description: 'The branch, tag or SHA that was checked out'
|
description: 'The branch, tag or SHA that was checked out'
|
||||||
commit:
|
commit:
|
||||||
description: 'The commit SHA that was checked out'
|
description: 'The commit SHA that was checked out'
|
||||||
|
cache-hit:
|
||||||
|
description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)'
|
||||||
runs:
|
runs:
|
||||||
using: node24
|
using: node24
|
||||||
main: dist/index.js
|
main: dist/index.js
|
||||||
|
|
|
||||||
726
dist/index.js
vendored
726
dist/index.js
vendored
|
|
@ -31883,6 +31883,64 @@ function qstring(str) {
|
||||||
/******/ }
|
/******/ }
|
||||||
/******/
|
/******/
|
||||||
/************************************************************************/
|
/************************************************************************/
|
||||||
|
/******/ /* webpack/runtime/create fake namespace object */
|
||||||
|
/******/ (() => {
|
||||||
|
/******/ var getProto = Object.getPrototypeOf ? (obj) => (Object.getPrototypeOf(obj)) : (obj) => (obj.__proto__);
|
||||||
|
/******/ var leafPrototypes;
|
||||||
|
/******/ // create a fake namespace object
|
||||||
|
/******/ // mode & 1: value is a module id, require it
|
||||||
|
/******/ // mode & 2: merge all properties of value into the ns
|
||||||
|
/******/ // mode & 4: return value when already ns object
|
||||||
|
/******/ // mode & 16: return value when it's Promise-like
|
||||||
|
/******/ // mode & 8|1: behave like require
|
||||||
|
/******/ __nccwpck_require__.t = function(value, mode) {
|
||||||
|
/******/ if(mode & 1) value = this(value);
|
||||||
|
/******/ if(mode & 8) return value;
|
||||||
|
/******/ if(typeof value === 'object' && value) {
|
||||||
|
/******/ if((mode & 4) && value.__esModule) return value;
|
||||||
|
/******/ if((mode & 16) && typeof value.then === 'function') return value;
|
||||||
|
/******/ }
|
||||||
|
/******/ var ns = Object.create(null);
|
||||||
|
/******/ __nccwpck_require__.r(ns);
|
||||||
|
/******/ var def = {};
|
||||||
|
/******/ leafPrototypes = leafPrototypes || [null, getProto({}), getProto([]), getProto(getProto)];
|
||||||
|
/******/ for(var current = mode & 2 && value; typeof current == 'object' && !~leafPrototypes.indexOf(current); current = getProto(current)) {
|
||||||
|
/******/ Object.getOwnPropertyNames(current).forEach((key) => (def[key] = () => (value[key])));
|
||||||
|
/******/ }
|
||||||
|
/******/ def['default'] = () => (value);
|
||||||
|
/******/ __nccwpck_require__.d(ns, def);
|
||||||
|
/******/ return ns;
|
||||||
|
/******/ };
|
||||||
|
/******/ })();
|
||||||
|
/******/
|
||||||
|
/******/ /* webpack/runtime/define property getters */
|
||||||
|
/******/ (() => {
|
||||||
|
/******/ // define getter functions for harmony exports
|
||||||
|
/******/ __nccwpck_require__.d = (exports, definition) => {
|
||||||
|
/******/ for(var key in definition) {
|
||||||
|
/******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) {
|
||||||
|
/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });
|
||||||
|
/******/ }
|
||||||
|
/******/ }
|
||||||
|
/******/ };
|
||||||
|
/******/ })();
|
||||||
|
/******/
|
||||||
|
/******/ /* webpack/runtime/hasOwnProperty shorthand */
|
||||||
|
/******/ (() => {
|
||||||
|
/******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))
|
||||||
|
/******/ })();
|
||||||
|
/******/
|
||||||
|
/******/ /* webpack/runtime/make namespace object */
|
||||||
|
/******/ (() => {
|
||||||
|
/******/ // define __esModule on exports
|
||||||
|
/******/ __nccwpck_require__.r = (exports) => {
|
||||||
|
/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) {
|
||||||
|
/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
||||||
|
/******/ }
|
||||||
|
/******/ Object.defineProperty(exports, '__esModule', { value: true });
|
||||||
|
/******/ };
|
||||||
|
/******/ })();
|
||||||
|
/******/
|
||||||
/******/ /* webpack/runtime/compat */
|
/******/ /* webpack/runtime/compat */
|
||||||
/******/
|
/******/
|
||||||
/******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/";
|
/******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/";
|
||||||
|
|
@ -32063,8 +32121,10 @@ function file_command_prepareKeyValueMessage(key, value) {
|
||||||
const external_path_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("path");
|
const external_path_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("path");
|
||||||
// EXTERNAL MODULE: external "http"
|
// EXTERNAL MODULE: external "http"
|
||||||
var external_http_ = __nccwpck_require__(8611);
|
var external_http_ = __nccwpck_require__(8611);
|
||||||
|
var external_http_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_http_, 2);
|
||||||
// EXTERNAL MODULE: external "https"
|
// EXTERNAL MODULE: external "https"
|
||||||
var external_https_ = __nccwpck_require__(5692);
|
var external_https_ = __nccwpck_require__(5692);
|
||||||
|
var external_https_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_https_, 2);
|
||||||
;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/proxy.js
|
;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/proxy.js
|
||||||
function getProxyUrl(reqUrl) {
|
function getProxyUrl(reqUrl) {
|
||||||
const usingSsl = reqUrl.protocol === 'https:';
|
const usingSsl = reqUrl.protocol === 'https:';
|
||||||
|
|
@ -32157,7 +32217,7 @@ class DecodedURL extends URL {
|
||||||
}
|
}
|
||||||
//# sourceMappingURL=proxy.js.map
|
//# sourceMappingURL=proxy.js.map
|
||||||
// EXTERNAL MODULE: ./node_modules/tunnel/index.js
|
// EXTERNAL MODULE: ./node_modules/tunnel/index.js
|
||||||
var node_modules_tunnel = __nccwpck_require__(770);
|
var tunnel = __nccwpck_require__(770);
|
||||||
// EXTERNAL MODULE: ./node_modules/undici/index.js
|
// EXTERNAL MODULE: ./node_modules/undici/index.js
|
||||||
var undici = __nccwpck_require__(6752);
|
var undici = __nccwpck_require__(6752);
|
||||||
;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/index.js
|
;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/index.js
|
||||||
|
|
@ -32235,7 +32295,7 @@ const HttpResponseRetryCodes = [
|
||||||
HttpCodes.ServiceUnavailable,
|
HttpCodes.ServiceUnavailable,
|
||||||
HttpCodes.GatewayTimeout
|
HttpCodes.GatewayTimeout
|
||||||
];
|
];
|
||||||
const RetryableHttpVerbs = (/* unused pure expression or super */ null && (['OPTIONS', 'GET', 'DELETE', 'HEAD']));
|
const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD'];
|
||||||
const ExponentialBackoffCeiling = 10;
|
const ExponentialBackoffCeiling = 10;
|
||||||
const ExponentialBackoffTimeSlice = 5;
|
const ExponentialBackoffTimeSlice = 5;
|
||||||
class HttpClientError extends Error {
|
class HttpClientError extends Error {
|
||||||
|
|
@ -32584,7 +32644,7 @@ class lib_HttpClient {
|
||||||
}
|
}
|
||||||
getAgentDispatcher(serverUrl) {
|
getAgentDispatcher(serverUrl) {
|
||||||
const parsedUrl = new URL(serverUrl);
|
const parsedUrl = new URL(serverUrl);
|
||||||
const proxyUrl = pm.getProxyUrl(parsedUrl);
|
const proxyUrl = getProxyUrl(parsedUrl);
|
||||||
const useProxy = proxyUrl && proxyUrl.hostname;
|
const useProxy = proxyUrl && proxyUrl.hostname;
|
||||||
if (!useProxy) {
|
if (!useProxy) {
|
||||||
return;
|
return;
|
||||||
|
|
@ -32595,7 +32655,7 @@ class lib_HttpClient {
|
||||||
const info = {};
|
const info = {};
|
||||||
info.parsedUrl = requestUrl;
|
info.parsedUrl = requestUrl;
|
||||||
const usingSsl = info.parsedUrl.protocol === 'https:';
|
const usingSsl = info.parsedUrl.protocol === 'https:';
|
||||||
info.httpModule = usingSsl ? https : http;
|
info.httpModule = usingSsl ? external_https_namespaceObject : external_http_namespaceObject;
|
||||||
const defaultPort = usingSsl ? 443 : 80;
|
const defaultPort = usingSsl ? 443 : 80;
|
||||||
info.options = {};
|
info.options = {};
|
||||||
info.options.host = info.parsedUrl.hostname;
|
info.options.host = info.parsedUrl.hostname;
|
||||||
|
|
@ -32694,7 +32754,7 @@ class lib_HttpClient {
|
||||||
}
|
}
|
||||||
_getAgent(parsedUrl) {
|
_getAgent(parsedUrl) {
|
||||||
let agent;
|
let agent;
|
||||||
const proxyUrl = pm.getProxyUrl(parsedUrl);
|
const proxyUrl = getProxyUrl(parsedUrl);
|
||||||
const useProxy = proxyUrl && proxyUrl.hostname;
|
const useProxy = proxyUrl && proxyUrl.hostname;
|
||||||
if (this._keepAlive && useProxy) {
|
if (this._keepAlive && useProxy) {
|
||||||
agent = this._proxyAgent;
|
agent = this._proxyAgent;
|
||||||
|
|
@ -32709,7 +32769,7 @@ class lib_HttpClient {
|
||||||
const usingSsl = parsedUrl.protocol === 'https:';
|
const usingSsl = parsedUrl.protocol === 'https:';
|
||||||
let maxSockets = 100;
|
let maxSockets = 100;
|
||||||
if (this.requestOptions) {
|
if (this.requestOptions) {
|
||||||
maxSockets = this.requestOptions.maxSockets || http.globalAgent.maxSockets;
|
maxSockets = this.requestOptions.maxSockets || external_http_.globalAgent.maxSockets;
|
||||||
}
|
}
|
||||||
// This is `useProxy` again, but we need to check `proxyURl` directly for TypeScripts's flow analysis.
|
// This is `useProxy` again, but we need to check `proxyURl` directly for TypeScripts's flow analysis.
|
||||||
if (proxyUrl && proxyUrl.hostname) {
|
if (proxyUrl && proxyUrl.hostname) {
|
||||||
|
|
@ -32734,7 +32794,7 @@ class lib_HttpClient {
|
||||||
// if tunneling agent isn't assigned create a new agent
|
// if tunneling agent isn't assigned create a new agent
|
||||||
if (!agent) {
|
if (!agent) {
|
||||||
const options = { keepAlive: this._keepAlive, maxSockets };
|
const options = { keepAlive: this._keepAlive, maxSockets };
|
||||||
agent = usingSsl ? new https.Agent(options) : new http.Agent(options);
|
agent = usingSsl ? new external_https_.Agent(options) : new external_http_.Agent(options);
|
||||||
this._agent = agent;
|
this._agent = agent;
|
||||||
}
|
}
|
||||||
if (usingSsl && this._ignoreSslError) {
|
if (usingSsl && this._ignoreSslError) {
|
||||||
|
|
@ -32757,7 +32817,7 @@ class lib_HttpClient {
|
||||||
return proxyAgent;
|
return proxyAgent;
|
||||||
}
|
}
|
||||||
const usingSsl = parsedUrl.protocol === 'https:';
|
const usingSsl = parsedUrl.protocol === 'https:';
|
||||||
proxyAgent = new ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && {
|
proxyAgent = new undici.ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && {
|
||||||
token: `Basic ${Buffer.from(`${proxyUrl.username}:${proxyUrl.password}`).toString('base64')}`
|
token: `Basic ${Buffer.from(`${proxyUrl.username}:${proxyUrl.password}`).toString('base64')}`
|
||||||
})));
|
})));
|
||||||
this._proxyAgentDispatcher = proxyAgent;
|
this._proxyAgentDispatcher = proxyAgent;
|
||||||
|
|
@ -41692,6 +41752,617 @@ function ref_helper_select(obj, path) {
|
||||||
return ref_helper_select(obj[key], path.substr(i + 1));
|
return ref_helper_select(obj[key], path.substr(i + 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
;// CONCATENATED MODULE: external "stream/promises"
|
||||||
|
const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises");
|
||||||
|
;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts
|
||||||
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
|
|
||||||
|
// Client for backend-core's git-mirror endpoints, authed by the runner verification
|
||||||
|
// token. The action seeds from a bare base mirror + a per-branch delta bundle (both
|
||||||
|
// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed
|
||||||
|
// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call
|
||||||
|
// fails closed to stock checkout.
|
||||||
|
const API_TIMEOUT_MS = 10_000;
|
||||||
|
function backend_api_baseUrl() {
|
||||||
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
|
||||||
|
}
|
||||||
|
function authHeader() {
|
||||||
|
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`;
|
||||||
|
}
|
||||||
|
function backend_api_endpoint(p) {
|
||||||
|
return `${backend_api_baseUrl()}/api/v1/git-mirrors/${p}`;
|
||||||
|
}
|
||||||
|
async function lookupRestore(repoKey, ref) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${backend_api_endpoint('restore-url')}?repo_key=${encodeURIComponent(repoKey)}&ref=${encodeURIComponent(ref)}`, {
|
||||||
|
headers: { authorization: authHeader() },
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
});
|
||||||
|
if (res.status === 200) {
|
||||||
|
const body = (await res.json());
|
||||||
|
return { kind: 'restore', base: body.base, branch: body.branch ?? null };
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
return { kind: 'cold' };
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core_debug('[wb-cache] restore-url answered 403 (disabled)');
|
||||||
|
return { kind: 'disabled' };
|
||||||
|
}
|
||||||
|
core_debug(`[wb-cache] restore-url answered ${res.status}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`[wb-cache] restore-url failed: ${error}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function requestUpload(p, body) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(backend_api_endpoint(p), {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: authHeader(),
|
||||||
|
'content-type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
});
|
||||||
|
if (res.status === 200) {
|
||||||
|
const b = (await res.json());
|
||||||
|
return { kind: 'grant', url: b.url };
|
||||||
|
}
|
||||||
|
if (res.status === 409) {
|
||||||
|
return { kind: 'locked' };
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
return { kind: 'disabled' };
|
||||||
|
}
|
||||||
|
core_debug(`[wb-cache] ${p} answered ${res.status}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`[wb-cache] ${p} failed: ${error}`);
|
||||||
|
return { kind: 'error' };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Cold repo: request the single-flight grant to build + upload the base mirror.
|
||||||
|
async function requestBaseUpload(repoKey) {
|
||||||
|
return requestUpload('base/upload-url', { repo_key: repoKey });
|
||||||
|
}
|
||||||
|
// Warm repo: request the grant to overwrite this branch's delta bundle.
|
||||||
|
async function requestBranchUpload(repoKey, ref, sha) {
|
||||||
|
return requestUpload('branch/upload-url', { repo_key: repoKey, ref, sha });
|
||||||
|
}
|
||||||
|
|
||||||
|
;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts
|
||||||
|
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
|
||||||
|
// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
|
||||||
|
// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
|
||||||
|
// so order-free); a cold repo single-flights the base build. Fail-open: any error →
|
||||||
|
// standard checkout.
|
||||||
|
const UPLOAD_TIMEOUT_MS = 15 * 60_000;
|
||||||
|
// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a
|
||||||
|
// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a
|
||||||
|
// WAN); parallel range GETs reach link rate (~200 MB/s).
|
||||||
|
const SEGMENT_SIZE = 4 * 1024 * 1024;
|
||||||
|
const SEGMENT_CONCURRENCY = 8;
|
||||||
|
const SEGMENT_TIMEOUT_MS = 30_000;
|
||||||
|
const SEGMENT_RETRIES = 5;
|
||||||
|
// Logged at info when off; the WARPBUILD_* env is only present on our runners.
|
||||||
|
const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)';
|
||||||
|
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
|
||||||
|
// Seeded bundles land here, out of the user's ref space; excluded objects for deltas.
|
||||||
|
const BASE_REFNS = 'refs/wb/base';
|
||||||
|
const BRANCH_REFNS = 'refs/wb/branch';
|
||||||
|
const UPLOAD_TIP_REF = 'refs/wb/tip';
|
||||||
|
let plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
|
// Null = attempt the cache; else a reason to log. We engage only where the result matches
|
||||||
|
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
|
||||||
|
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
|
||||||
|
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
|
||||||
|
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
|
||||||
|
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
|
||||||
|
// exactly as upstream — the mirror only accelerates the git-object half.
|
||||||
|
function getMirrorCacheSkipReason(settings) {
|
||||||
|
if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
||||||
|
!process.env['WARPBUILD_HOST_URL']) {
|
||||||
|
return SKIP_NOT_WARPBUILD;
|
||||||
|
}
|
||||||
|
if (!process.env['GITHUB_REPOSITORY_ID']) {
|
||||||
|
return 'GITHUB_REPOSITORY_ID is not set';
|
||||||
|
}
|
||||||
|
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`;
|
||||||
|
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
|
||||||
|
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`;
|
||||||
|
}
|
||||||
|
const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, '');
|
||||||
|
if (server !== 'https://github.com') {
|
||||||
|
return `server '${server}' is not github.com`;
|
||||||
|
}
|
||||||
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
|
return 'no exact commit sha to key on';
|
||||||
|
}
|
||||||
|
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
|
||||||
|
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
|
||||||
|
if (settings.fetchDepth > 1) {
|
||||||
|
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`;
|
||||||
|
}
|
||||||
|
if (settings.filter) {
|
||||||
|
return 'a fetch filter is configured';
|
||||||
|
}
|
||||||
|
if (settings.sparseCheckout) {
|
||||||
|
return 'sparse checkout is configured';
|
||||||
|
}
|
||||||
|
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// The durable branch to key the per-branch bundle on. For pull_request events the merge
|
||||||
|
// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for
|
||||||
|
// tags / detached HEAD → seed the base only, upload nothing.
|
||||||
|
function computeRefKey(settings) {
|
||||||
|
const baseRef = process.env['GITHUB_BASE_REF']; // set on pull_request events
|
||||||
|
if (baseRef) {
|
||||||
|
return baseRef;
|
||||||
|
}
|
||||||
|
const ref = process.env['GITHUB_REF'] || settings.ref || '';
|
||||||
|
if (ref.startsWith('refs/heads/')) {
|
||||||
|
return ref.substring('refs/heads/'.length);
|
||||||
|
}
|
||||||
|
if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
// Already a short branch name, or empty.
|
||||||
|
return SHA_PATTERN.test(ref) ? '' : ref;
|
||||||
|
}
|
||||||
|
// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps
|
||||||
|
// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache).
|
||||||
|
async function setup(settings) {
|
||||||
|
const mode = await setupImpl(settings);
|
||||||
|
setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false');
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
|
||||||
|
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
|
||||||
|
// false since we're no longer serving from cache.
|
||||||
|
async function abandon(settings) {
|
||||||
|
plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
|
setOutput('cache-hit', 'false');
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
|
async function setupImpl(settings) {
|
||||||
|
plan = { mode: 'off', repoKey: '', refKey: '' };
|
||||||
|
const skipReason = getMirrorCacheSkipReason(settings);
|
||||||
|
if (skipReason) {
|
||||||
|
info(`WarpBuild mirror cache skipped: ${skipReason}`);
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
startGroup('WarpBuild: checkout mirror cache');
|
||||||
|
try {
|
||||||
|
return await setupInner(settings);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`);
|
||||||
|
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
|
||||||
|
// clean repo (objects just dangle harmlessly).
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
endGroup();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function setupInner(settings) {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
|
||||||
|
const refKey = computeRefKey(settings);
|
||||||
|
plan = { mode: 'off', repoKey, refKey };
|
||||||
|
const lookup = await lookupRestore(repoKey, refKey);
|
||||||
|
if (lookup.kind === 'disabled') {
|
||||||
|
info('Mirror cache is disabled by the backend for this organization');
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'error') {
|
||||||
|
info('Mirror cache backend unavailable; using standard checkout');
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'cold') {
|
||||||
|
const grant = await requestBaseUpload(repoKey);
|
||||||
|
if (grant.kind === 'grant') {
|
||||||
|
info('Cold repo: building the base mirror this run (full fetch, then upload)');
|
||||||
|
plan = { mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url };
|
||||||
|
return 'cold-build';
|
||||||
|
}
|
||||||
|
info('Cold repo: base build already in progress elsewhere; using standard checkout');
|
||||||
|
return 'off';
|
||||||
|
}
|
||||||
|
// Restore: seed the base, then this branch's delta if present.
|
||||||
|
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS);
|
||||||
|
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
|
||||||
|
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
|
||||||
|
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
|
||||||
|
// (depth 1) populates only the target ref, so there we keep the base internal and let the
|
||||||
|
// delta fetch provide just that ref.
|
||||||
|
if (settings.fetchDepth <= 0) {
|
||||||
|
await exposeBaseRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
|
if (lookup.branch) {
|
||||||
|
try {
|
||||||
|
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
info(`Branch delta seed skipped (${error}); base only`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info(`Seeded base mirror${lookup.branch ? ' + branch delta' : ''}; the GitHub fetch will be a tip delta`);
|
||||||
|
plan = { mode: 'seeded', repoKey, refKey };
|
||||||
|
return 'seeded';
|
||||||
|
}
|
||||||
|
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
|
||||||
|
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
|
||||||
|
// try/catch for normal errors, plus scoped process guards (installed only for the upload
|
||||||
|
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
|
||||||
|
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
|
||||||
|
// exit(0). Safe because nothing checkout-relevant runs after this point.
|
||||||
|
async function contribute(settings) {
|
||||||
|
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const guard = (error) => {
|
||||||
|
warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`);
|
||||||
|
process.exit(0);
|
||||||
|
};
|
||||||
|
process.on('uncaughtException', guard);
|
||||||
|
process.on('unhandledRejection', guard);
|
||||||
|
try {
|
||||||
|
if (plan.mode === 'cold-build') {
|
||||||
|
await uploadBaseMirror(settings);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
await uploadBranchDelta(settings);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror upload skipped: ${error}`);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
|
||||||
|
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
|
||||||
|
if (plan.mode === 'seeded') {
|
||||||
|
await cleanupWbRefs(settings.repositoryPath);
|
||||||
|
}
|
||||||
|
process.removeListener('uncaughtException', guard);
|
||||||
|
process.removeListener('unhandledRejection', guard);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
|
||||||
|
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
|
||||||
|
// nothing is extracted into .git directly.
|
||||||
|
async function seedBundle(repoPath, url, refNs) {
|
||||||
|
const tmp = tempBundlePath('seed');
|
||||||
|
const what = refNs === BASE_REFNS ? 'base' : 'branch';
|
||||||
|
try {
|
||||||
|
await downloadTo(url, tmp, what);
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'fetch',
|
||||||
|
'--quiet',
|
||||||
|
'--no-tags',
|
||||||
|
tmp,
|
||||||
|
`+refs/*:${refNs}/*`
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Copy the seeded base's branches + tags out to the namespaces upstream uses
|
||||||
|
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
|
||||||
|
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
|
||||||
|
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
|
||||||
|
async function exposeBaseRefs(repoPath) {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'for-each-ref',
|
||||||
|
'--format=create refs/%(refname:lstrip=3) %(objectname)',
|
||||||
|
`${BASE_REFNS}/remotes/origin`,
|
||||||
|
`${BASE_REFNS}/tags`
|
||||||
|
], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
if (!out.trim()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
|
||||||
|
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
|
||||||
|
async function cleanupWbRefs(repoPath) {
|
||||||
|
try {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
if (!out.trim()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
core_debug(`WarpBuild ref cleanup skipped: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Cold-build: after the forced full fetch the repo holds all branches under
|
||||||
|
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
||||||
|
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
||||||
|
async function uploadBaseMirror(settings) {
|
||||||
|
if (!plan.baseUploadUrl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const tmp = tempBundlePath('base');
|
||||||
|
try {
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
settings.repositoryPath,
|
||||||
|
'bundle',
|
||||||
|
'create',
|
||||||
|
tmp,
|
||||||
|
'--remotes=origin',
|
||||||
|
'--tags'
|
||||||
|
]);
|
||||||
|
await httpPut(plan.baseUploadUrl, tmp);
|
||||||
|
info('Uploaded base mirror');
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Seeded: build the base-relative delta (target sha, excluding everything in the base)
|
||||||
|
// and overwrite this branch's bundle. Guarded by a per-branch lock server-side.
|
||||||
|
async function uploadBranchDelta(settings) {
|
||||||
|
if (!plan.refKey) {
|
||||||
|
return; // tag / detached HEAD: nothing to roll
|
||||||
|
}
|
||||||
|
const repoPath = settings.repositoryPath;
|
||||||
|
const sha = settings.commit;
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]);
|
||||||
|
try {
|
||||||
|
if (!(await hasBaseRefs(repoPath))) {
|
||||||
|
info('No base refs to diff against; branch delta skipped');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// If the tip is already in the base there is no delta to roll, and `git bundle
|
||||||
|
// create` errors on an empty range ("Refusing to create empty bundle"). Detect it
|
||||||
|
// first — and before taking the server lock, so an empty delta never blocks the
|
||||||
|
// other jobs racing the same branch.
|
||||||
|
if (await branchDeltaIsEmpty(repoPath)) {
|
||||||
|
info('Tip already contained in the base; no branch delta to upload');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha);
|
||||||
|
if (grant.kind !== 'grant') {
|
||||||
|
info(`Branch delta upload skipped (${grant.kind})`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const tmp = tempBundlePath('branch');
|
||||||
|
try {
|
||||||
|
// Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo
|
||||||
|
// has hundreds of base refs, and that many args overflows the Windows command-line
|
||||||
|
// limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the
|
||||||
|
// same base-relative (order-free) delta.
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'bundle',
|
||||||
|
'create',
|
||||||
|
tmp,
|
||||||
|
UPLOAD_TIP_REF,
|
||||||
|
'--not',
|
||||||
|
`--glob=${BASE_REFNS}/*`
|
||||||
|
]);
|
||||||
|
await httpPut(grant.url, tmp);
|
||||||
|
info(`Uploaded branch delta for '${plan.refKey}'`);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await external_fs_namespaceObject.promises.rm(tmp, { force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], {
|
||||||
|
ignoreReturnCode: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The tip's delta against the base is empty when the tip is already reachable from the
|
||||||
|
// base — nothing new to bundle.
|
||||||
|
async function branchDeltaIsEmpty(repoPath) {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'rev-list',
|
||||||
|
'--count',
|
||||||
|
UPLOAD_TIP_REF,
|
||||||
|
'--not',
|
||||||
|
`--glob=${BASE_REFNS}/*`
|
||||||
|
], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
return out.trim() === '0';
|
||||||
|
}
|
||||||
|
// Whether any base ref was seeded — the guard that keeps the delta base-relative. The
|
||||||
|
// base is excluded by glob (see uploadBranchDelta), so we only need existence here.
|
||||||
|
async function hasBaseRefs(repoPath) {
|
||||||
|
let out = '';
|
||||||
|
await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } });
|
||||||
|
return out.trim().length > 0;
|
||||||
|
}
|
||||||
|
// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle
|
||||||
|
// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling.
|
||||||
|
// Falls back to a single stream when the server doesn't support ranges.
|
||||||
|
async function downloadTo(url, dest, what) {
|
||||||
|
const started = Date.now();
|
||||||
|
// Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes.
|
||||||
|
const http = new lib_HttpClient('warpbuild-checkout-mirror', undefined, {
|
||||||
|
socketTimeout: SEGMENT_TIMEOUT_MS,
|
||||||
|
keepAlive: true
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const total = await probeRangeSize(http, url);
|
||||||
|
if (total === null) {
|
||||||
|
await downloadSingleStream(http, url, dest);
|
||||||
|
const bytes = (await external_fs_namespaceObject.promises.stat(dest)).size;
|
||||||
|
info(`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const segments = [];
|
||||||
|
for (let off = 0; off < total; off += SEGMENT_SIZE) {
|
||||||
|
segments.push([off, Math.min(SEGMENT_SIZE, total - off)]);
|
||||||
|
}
|
||||||
|
const width = Math.min(SEGMENT_CONCURRENCY, segments.length);
|
||||||
|
const fh = await external_fs_namespaceObject.promises.open(dest, 'w');
|
||||||
|
try {
|
||||||
|
let next = 0;
|
||||||
|
const worker = async () => {
|
||||||
|
for (;;) {
|
||||||
|
const i = next++;
|
||||||
|
if (i >= segments.length) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const [offset, count] = segments[i];
|
||||||
|
const buf = await downloadSegment(http, url, offset, count);
|
||||||
|
await fh.write(buf, 0, count, offset);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const pool = [];
|
||||||
|
for (let k = 0; k < width; k++) {
|
||||||
|
pool.push(worker());
|
||||||
|
}
|
||||||
|
await Promise.all(pool);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
await fh.close();
|
||||||
|
}
|
||||||
|
info(`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`);
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
http.dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Size + elapsed/throughput for the mirror download o11y line.
|
||||||
|
function fmtSize(bytes) {
|
||||||
|
return `${(bytes / 1e6).toFixed(1)} MB`;
|
||||||
|
}
|
||||||
|
function fmtElapsed(startedMs, bytes) {
|
||||||
|
const secs = (Date.now() - startedMs) / 1000;
|
||||||
|
const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a';
|
||||||
|
return `${secs.toFixed(2)}s (${rate} MB/s)`;
|
||||||
|
}
|
||||||
|
// GET bytes=0-0 to learn the total size and confirm range support; null → not supported.
|
||||||
|
async function probeRangeSize(http, url) {
|
||||||
|
const res = await http.get(url, { Range: 'bytes=0-0' });
|
||||||
|
if (res.message.statusCode !== 206) {
|
||||||
|
res.message.destroy(); // no range support; don't read a full 200 body into memory
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
await res.readBody(); // 1 byte; return the socket to the keep-alive pool
|
||||||
|
const contentRange = res.message.headers['content-range'];
|
||||||
|
const match = typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null;
|
||||||
|
const total = match ? parseInt(match[1], 10) : NaN;
|
||||||
|
return Number.isFinite(total) && total > 0 ? total : null;
|
||||||
|
}
|
||||||
|
// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes.
|
||||||
|
async function downloadSegment(http, url, offset, count) {
|
||||||
|
let lastErr;
|
||||||
|
for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) {
|
||||||
|
try {
|
||||||
|
const res = await http.get(url, {
|
||||||
|
Range: `bytes=${offset}-${offset + count - 1}`
|
||||||
|
});
|
||||||
|
if (res.message.statusCode !== 206) {
|
||||||
|
await res.readBody(); // drain before retrying so the socket is reusable
|
||||||
|
throw new Error(`range GET answered ${res.message.statusCode}`);
|
||||||
|
}
|
||||||
|
const buf = await res.readBodyBuffer?.();
|
||||||
|
if (!buf) {
|
||||||
|
throw new Error('http-client response lacks readBodyBuffer');
|
||||||
|
}
|
||||||
|
if (buf.length !== count) {
|
||||||
|
throw new Error(`short segment at ${offset}: ${buf.length}/${count}`);
|
||||||
|
}
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
lastErr = error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastErr instanceof Error
|
||||||
|
? lastErr
|
||||||
|
: new Error(`segment at ${offset} failed`);
|
||||||
|
}
|
||||||
|
async function downloadSingleStream(http, url, dest) {
|
||||||
|
const res = await http.get(url);
|
||||||
|
const status = res.message.statusCode ?? 0;
|
||||||
|
if (status < 200 || status >= 300) {
|
||||||
|
await res.readBody();
|
||||||
|
throw new Error(`download answered ${status}`);
|
||||||
|
}
|
||||||
|
await (0,promises_namespaceObject.pipeline)(res.message, external_fs_namespaceObject.createWriteStream(dest));
|
||||||
|
}
|
||||||
|
async function httpPut(url, file) {
|
||||||
|
const stat = await external_fs_namespaceObject.promises.stat(file);
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: 'PUT',
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body
|
||||||
|
body: external_fs_namespaceObject.createReadStream(file),
|
||||||
|
duplex: 'half',
|
||||||
|
headers: { 'content-length': String(stat.size) },
|
||||||
|
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`PUT answered ${res.status}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a
|
||||||
|
// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file).
|
||||||
|
let mirrorTmpDir;
|
||||||
|
function tempBundlePath(tag) {
|
||||||
|
if (!mirrorTmpDir) {
|
||||||
|
mirrorTmpDir = external_fs_namespaceObject.mkdtempSync(external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), 'wb-mirror-'));
|
||||||
|
}
|
||||||
|
return external_path_namespaceObject.join(mirrorTmpDir, `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`);
|
||||||
|
}
|
||||||
|
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
||||||
|
async function resetGitObjects(gitDir) {
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true
|
||||||
|
});
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'shallow'), { force: true });
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`.
|
||||||
|
async function hasGit() {
|
||||||
|
return Boolean(await io.which('git', false));
|
||||||
|
}
|
||||||
|
|
||||||
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
;// CONCATENATED MODULE: ./src/git-source-provider.ts
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -41705,6 +42376,7 @@ function ref_helper_select(obj, path) {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function getSource(settings) {
|
async function getSource(settings) {
|
||||||
// Repository URL
|
// Repository URL
|
||||||
info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`);
|
info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`);
|
||||||
|
|
@ -41724,6 +42396,7 @@ async function getSource(settings) {
|
||||||
const git = await getGitCommandManager(settings);
|
const git = await getGitCommandManager(settings);
|
||||||
endGroup();
|
endGroup();
|
||||||
let authHelper = null;
|
let authHelper = null;
|
||||||
|
let warpbuildMode = 'off';
|
||||||
try {
|
try {
|
||||||
if (git) {
|
if (git) {
|
||||||
authHelper = createAuthHelper(git, settings);
|
authHelper = createAuthHelper(git, settings);
|
||||||
|
|
@ -41774,6 +42447,8 @@ async function getSource(settings) {
|
||||||
await git.init(objectFormat);
|
await git.init(objectFormat);
|
||||||
await git.remoteAdd('origin', repositoryUrl);
|
await git.remoteAdd('origin', repositoryUrl);
|
||||||
endGroup();
|
endGroup();
|
||||||
|
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
|
||||||
|
warpbuildMode = await setup(settings);
|
||||||
}
|
}
|
||||||
// Disable automatic garbage collection
|
// Disable automatic garbage collection
|
||||||
startGroup('Disabling automatic garbage collection');
|
startGroup('Disabling automatic garbage collection');
|
||||||
|
|
@ -41813,7 +42488,30 @@ async function getSource(settings) {
|
||||||
else if (settings.sparseCheckout) {
|
else if (settings.sparseCheckout) {
|
||||||
fetchOptions.filter = 'blob:none';
|
fetchOptions.filter = 'blob:none';
|
||||||
}
|
}
|
||||||
if (settings.fetchDepth <= 0) {
|
let warpbuildFetchDone = false;
|
||||||
|
if (warpbuildMode === 'seeded') {
|
||||||
|
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
||||||
|
// objects. No depth — the base is full history, so this stays non-shallow. If it
|
||||||
|
// fails for any reason, disengage cleanly and fall through to the standard fetch.
|
||||||
|
try {
|
||||||
|
const refSpec = getRefSpec(settings.ref, settings.commit);
|
||||||
|
await git.fetch(refSpec, fetchOptions);
|
||||||
|
if (!(await testRef(git, settings.ref, settings.commit))) {
|
||||||
|
throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
||||||
|
`The ref may have been updated after the workflow was triggered.`);
|
||||||
|
}
|
||||||
|
warpbuildFetchDone = true;
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`);
|
||||||
|
await abandon(settings);
|
||||||
|
warpbuildMode = 'off';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (warpbuildFetchDone) {
|
||||||
|
// The seeded delta fetch already brought in the target commit.
|
||||||
|
}
|
||||||
|
else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
|
let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit);
|
||||||
await git.fetch(refSpec, fetchOptions);
|
await git.fetch(refSpec, fetchOptions);
|
||||||
|
|
@ -41905,6 +42603,16 @@ async function getSource(settings) {
|
||||||
setOutput('commit', commitSHA.trim());
|
setOutput('commit', commitSHA.trim());
|
||||||
// Check for incorrect pull request merge commit
|
// Check for incorrect pull request merge commit
|
||||||
await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl);
|
await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl);
|
||||||
|
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
|
||||||
|
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
|
||||||
|
// scoped process guard it installs (which ignores any uncaught upload error now that
|
||||||
|
// the checkout is already complete).
|
||||||
|
try {
|
||||||
|
await contribute(settings);
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
warning(`WarpBuild mirror upload skipped: ${error}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
// Remove auth
|
// Remove auth
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@
|
||||||
"@actions/core": "^3.0.1",
|
"@actions/core": "^3.0.1",
|
||||||
"@actions/exec": "^3.0.0",
|
"@actions/exec": "^3.0.0",
|
||||||
"@actions/github": "^9.1.1",
|
"@actions/github": "^9.1.1",
|
||||||
|
"@actions/http-client": "^4.0.1",
|
||||||
"@actions/io": "^3.0.2",
|
"@actions/io": "^3.0.2",
|
||||||
"@actions/tool-cache": "^4.0.0"
|
"@actions/tool-cache": "^4.0.0"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import * as path from 'path'
|
||||||
import * as refHelper from './ref-helper.js'
|
import * as refHelper from './ref-helper.js'
|
||||||
import * as stateHelper from './state-helper.js'
|
import * as stateHelper from './state-helper.js'
|
||||||
import * as urlHelper from './url-helper.js'
|
import * as urlHelper from './url-helper.js'
|
||||||
|
import * as warpbuildMirror from './warpbuild/mirror-cache.js'
|
||||||
import {
|
import {
|
||||||
MinimumGitSparseCheckoutVersion,
|
MinimumGitSparseCheckoutVersion,
|
||||||
IGitCommandManager
|
IGitCommandManager
|
||||||
|
|
@ -40,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
let authHelper: gitAuthHelper.IGitAuthHelper | null = null
|
let authHelper: gitAuthHelper.IGitAuthHelper | null = null
|
||||||
|
let warpbuildMode: warpbuildMirror.MirrorMode = 'off'
|
||||||
try {
|
try {
|
||||||
if (git) {
|
if (git) {
|
||||||
authHelper = gitAuthHelper.createAuthHelper(git, settings)
|
authHelper = gitAuthHelper.createAuthHelper(git, settings)
|
||||||
|
|
@ -130,6 +132,9 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
await git.init(objectFormat)
|
await git.init(objectFormat)
|
||||||
await git.remoteAdd('origin', repositoryUrl)
|
await git.remoteAdd('origin', repositoryUrl)
|
||||||
core.endGroup()
|
core.endGroup()
|
||||||
|
|
||||||
|
// WarpBuild snapshot cache: hit = objects restored, fetch below is skipped
|
||||||
|
warpbuildMode = await warpbuildMirror.setup(settings)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Disable automatic garbage collection
|
// Disable automatic garbage collection
|
||||||
|
|
@ -185,7 +190,33 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
fetchOptions.filter = 'blob:none'
|
fetchOptions.filter = 'blob:none'
|
||||||
}
|
}
|
||||||
|
|
||||||
if (settings.fetchDepth <= 0) {
|
let warpbuildFetchDone = false
|
||||||
|
if (warpbuildMode === 'seeded') {
|
||||||
|
// Seeded from the mirror: fetch only the tip delta, negotiating against the seeded
|
||||||
|
// objects. No depth — the base is full history, so this stays non-shallow. If it
|
||||||
|
// fails for any reason, disengage cleanly and fall through to the standard fetch.
|
||||||
|
try {
|
||||||
|
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
|
||||||
|
await git.fetch(refSpec, fetchOptions)
|
||||||
|
if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
|
||||||
|
throw new Error(
|
||||||
|
`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` +
|
||||||
|
`The ref may have been updated after the workflow was triggered.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
warpbuildFetchDone = true
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`
|
||||||
|
)
|
||||||
|
await warpbuildMirror.abandon(settings)
|
||||||
|
warpbuildMode = 'off'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (warpbuildFetchDone) {
|
||||||
|
// The seeded delta fetch already brought in the target commit.
|
||||||
|
} else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) {
|
||||||
// Fetch all branches and tags
|
// Fetch all branches and tags
|
||||||
let refSpec = refHelper.getRefSpecForAllHistory(
|
let refSpec = refHelper.getRefSpecForAllHistory(
|
||||||
settings.ref,
|
settings.ref,
|
||||||
|
|
@ -313,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
|
||||||
settings.commit,
|
settings.commit,
|
||||||
settings.githubServerUrl
|
settings.githubServerUrl
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect
|
||||||
|
// the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the
|
||||||
|
// scoped process guard it installs (which ignores any uncaught upload error now that
|
||||||
|
// the checkout is already complete).
|
||||||
|
try {
|
||||||
|
await warpbuildMirror.contribute(settings)
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(`WarpBuild mirror upload skipped: ${error}`)
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
// Remove auth
|
// Remove auth
|
||||||
if (authHelper) {
|
if (authHelper) {
|
||||||
|
|
|
||||||
120
src/warpbuild/backend-api.ts
Normal file
120
src/warpbuild/backend-api.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
/* eslint-disable i18n-text/no-en -- upstream convention */
|
||||||
|
import * as core from '@actions/core'
|
||||||
|
|
||||||
|
// Client for backend-core's git-mirror endpoints, authed by the runner verification
|
||||||
|
// token. The action seeds from a bare base mirror + a per-branch delta bundle (both
|
||||||
|
// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed
|
||||||
|
// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call
|
||||||
|
// fails closed to stock checkout.
|
||||||
|
|
||||||
|
const API_TIMEOUT_MS = 10_000
|
||||||
|
|
||||||
|
export interface Presigned {
|
||||||
|
url: string
|
||||||
|
size_bytes?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET restore-url: what to seed before the delta fetch. 'cold' = no base yet.
|
||||||
|
export type RestoreLookup =
|
||||||
|
| {kind: 'restore'; base: Presigned; branch: Presigned | null}
|
||||||
|
| {kind: 'cold'}
|
||||||
|
| {kind: 'disabled'}
|
||||||
|
| {kind: 'error'}
|
||||||
|
|
||||||
|
// POST *upload-url: a presigned PUT, or a signal to skip (locked/disabled).
|
||||||
|
export type UploadGrant =
|
||||||
|
| {kind: 'grant'; url: string}
|
||||||
|
| {kind: 'locked'}
|
||||||
|
| {kind: 'disabled'}
|
||||||
|
| {kind: 'error'}
|
||||||
|
|
||||||
|
function baseUrl(): string {
|
||||||
|
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '')
|
||||||
|
}
|
||||||
|
|
||||||
|
function authHeader(): string {
|
||||||
|
return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function endpoint(p: string): string {
|
||||||
|
return `${baseUrl()}/api/v1/git-mirrors/${p}`
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function lookupRestore(
|
||||||
|
repoKey: string,
|
||||||
|
ref: string
|
||||||
|
): Promise<RestoreLookup> {
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`${endpoint('restore-url')}?repo_key=${encodeURIComponent(
|
||||||
|
repoKey
|
||||||
|
)}&ref=${encodeURIComponent(ref)}`,
|
||||||
|
{
|
||||||
|
headers: {authorization: authHeader()},
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (res.status === 200) {
|
||||||
|
const body = (await res.json()) as {
|
||||||
|
base: Presigned
|
||||||
|
branch: Presigned | null
|
||||||
|
}
|
||||||
|
return {kind: 'restore', base: body.base, branch: body.branch ?? null}
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
return {kind: 'cold'}
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
core.debug('[wb-cache] restore-url answered 403 (disabled)')
|
||||||
|
return {kind: 'disabled'}
|
||||||
|
}
|
||||||
|
core.debug(`[wb-cache] restore-url answered ${res.status}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`[wb-cache] restore-url failed: ${error}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestUpload(p: string, body: object): Promise<UploadGrant> {
|
||||||
|
try {
|
||||||
|
const res = await fetch(endpoint(p), {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: authHeader(),
|
||||||
|
'content-type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
signal: AbortSignal.timeout(API_TIMEOUT_MS)
|
||||||
|
})
|
||||||
|
if (res.status === 200) {
|
||||||
|
const b = (await res.json()) as {url: string}
|
||||||
|
return {kind: 'grant', url: b.url}
|
||||||
|
}
|
||||||
|
if (res.status === 409) {
|
||||||
|
return {kind: 'locked'}
|
||||||
|
}
|
||||||
|
if (res.status === 403) {
|
||||||
|
return {kind: 'disabled'}
|
||||||
|
}
|
||||||
|
core.debug(`[wb-cache] ${p} answered ${res.status}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`[wb-cache] ${p} failed: ${error}`)
|
||||||
|
return {kind: 'error'}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cold repo: request the single-flight grant to build + upload the base mirror.
|
||||||
|
export async function requestBaseUpload(repoKey: string): Promise<UploadGrant> {
|
||||||
|
return requestUpload('base/upload-url', {repo_key: repoKey})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warm repo: request the grant to overwrite this branch's delta bundle.
|
||||||
|
export async function requestBranchUpload(
|
||||||
|
repoKey: string,
|
||||||
|
ref: string,
|
||||||
|
sha: string
|
||||||
|
): Promise<UploadGrant> {
|
||||||
|
return requestUpload('branch/upload-url', {repo_key: repoKey, ref, sha})
|
||||||
|
}
|
||||||
625
src/warpbuild/mirror-cache.ts
Normal file
625
src/warpbuild/mirror-cache.ts
Normal file
|
|
@ -0,0 +1,625 @@
|
||||||
|
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
|
||||||
|
import * as core from '@actions/core'
|
||||||
|
import * as exec from '@actions/exec'
|
||||||
|
import * as io from '@actions/io'
|
||||||
|
import {HttpClient} from '@actions/http-client'
|
||||||
|
import * as crypto from 'crypto'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import * as os from 'os'
|
||||||
|
import * as path from 'path'
|
||||||
|
import {pipeline} from 'stream/promises'
|
||||||
|
import {IGitSourceSettings} from '../git-source-settings.js'
|
||||||
|
import * as api from './backend-api.js'
|
||||||
|
|
||||||
|
// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on
|
||||||
|
// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is
|
||||||
|
// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative,
|
||||||
|
// so order-free); a cold repo single-flights the base build. Fail-open: any error →
|
||||||
|
// standard checkout.
|
||||||
|
|
||||||
|
const UPLOAD_TIMEOUT_MS = 15 * 60_000
|
||||||
|
|
||||||
|
// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a
|
||||||
|
// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a
|
||||||
|
// WAN); parallel range GETs reach link rate (~200 MB/s).
|
||||||
|
const SEGMENT_SIZE = 4 * 1024 * 1024
|
||||||
|
const SEGMENT_CONCURRENCY = 8
|
||||||
|
const SEGMENT_TIMEOUT_MS = 30_000
|
||||||
|
const SEGMENT_RETRIES = 5
|
||||||
|
|
||||||
|
// Logged at info when off; the WARPBUILD_* env is only present on our runners.
|
||||||
|
export const SKIP_NOT_WARPBUILD =
|
||||||
|
'not running on a WarpBuild runner (WARPBUILD_* env not present)'
|
||||||
|
|
||||||
|
const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/
|
||||||
|
|
||||||
|
// Seeded bundles land here, out of the user's ref space; excluded objects for deltas.
|
||||||
|
const BASE_REFNS = 'refs/wb/base'
|
||||||
|
const BRANCH_REFNS = 'refs/wb/branch'
|
||||||
|
const UPLOAD_TIP_REF = 'refs/wb/tip'
|
||||||
|
|
||||||
|
export type MirrorMode = 'off' | 'seeded' | 'cold-build'
|
||||||
|
|
||||||
|
interface Plan {
|
||||||
|
mode: MirrorMode
|
||||||
|
repoKey: string
|
||||||
|
refKey: string
|
||||||
|
baseUploadUrl?: string // cold-build only
|
||||||
|
}
|
||||||
|
let plan: Plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
|
|
||||||
|
// Null = attempt the cache; else a reason to log. We engage only where the result matches
|
||||||
|
// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth
|
||||||
|
// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer
|
||||||
|
// to upstream. sparse/filter change the object set we model, so they defer too. LFS does
|
||||||
|
// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the
|
||||||
|
// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top,
|
||||||
|
// exactly as upstream — the mirror only accelerates the git-object half.
|
||||||
|
export function getMirrorCacheSkipReason(
|
||||||
|
settings: IGitSourceSettings
|
||||||
|
): string | null {
|
||||||
|
if (
|
||||||
|
!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] ||
|
||||||
|
!process.env['WARPBUILD_HOST_URL']
|
||||||
|
) {
|
||||||
|
return SKIP_NOT_WARPBUILD
|
||||||
|
}
|
||||||
|
if (!process.env['GITHUB_REPOSITORY_ID']) {
|
||||||
|
return 'GITHUB_REPOSITORY_ID is not set'
|
||||||
|
}
|
||||||
|
const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`
|
||||||
|
if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) {
|
||||||
|
return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`
|
||||||
|
}
|
||||||
|
const server = (settings.githubServerUrl || 'https://github.com').replace(
|
||||||
|
/\/+$/,
|
||||||
|
''
|
||||||
|
)
|
||||||
|
if (server !== 'https://github.com') {
|
||||||
|
return `server '${server}' is not github.com`
|
||||||
|
}
|
||||||
|
if (!settings.commit || !SHA_PATTERN.test(settings.commit)) {
|
||||||
|
return 'no exact commit sha to key on'
|
||||||
|
}
|
||||||
|
// Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the
|
||||||
|
// mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream.
|
||||||
|
if (settings.fetchDepth > 1) {
|
||||||
|
return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`
|
||||||
|
}
|
||||||
|
if (settings.filter) {
|
||||||
|
return 'a fetch filter is configured'
|
||||||
|
}
|
||||||
|
if (settings.sparseCheckout) {
|
||||||
|
return 'sparse checkout is configured'
|
||||||
|
}
|
||||||
|
// LFS intentionally does not skip — see the note above getMirrorCacheSkipReason.
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// The durable branch to key the per-branch bundle on. For pull_request events the merge
|
||||||
|
// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for
|
||||||
|
// tags / detached HEAD → seed the base only, upload nothing.
|
||||||
|
export function computeRefKey(settings: IGitSourceSettings): string {
|
||||||
|
const baseRef = process.env['GITHUB_BASE_REF'] // set on pull_request events
|
||||||
|
if (baseRef) {
|
||||||
|
return baseRef
|
||||||
|
}
|
||||||
|
const ref = process.env['GITHUB_REF'] || settings.ref || ''
|
||||||
|
if (ref.startsWith('refs/heads/')) {
|
||||||
|
return ref.substring('refs/heads/'.length)
|
||||||
|
}
|
||||||
|
if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) {
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
// Already a short branch name, or empty.
|
||||||
|
return SHA_PATTERN.test(ref) ? '' : ref
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps
|
||||||
|
// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache).
|
||||||
|
export async function setup(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
|
const mode = await setupImpl(settings)
|
||||||
|
core.setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false')
|
||||||
|
return mode
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so
|
||||||
|
// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to
|
||||||
|
// false since we're no longer serving from cache.
|
||||||
|
export async function abandon(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
|
core.setOutput('cache-hit', 'false')
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setupImpl(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
|
plan = {mode: 'off', repoKey: '', refKey: ''}
|
||||||
|
const skipReason = getMirrorCacheSkipReason(settings)
|
||||||
|
if (skipReason) {
|
||||||
|
core.info(`WarpBuild mirror cache skipped: ${skipReason}`)
|
||||||
|
return 'off'
|
||||||
|
}
|
||||||
|
core.startGroup('WarpBuild: checkout mirror cache')
|
||||||
|
try {
|
||||||
|
return await setupInner(settings)
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild mirror cache unavailable, using standard checkout: ${error}`
|
||||||
|
)
|
||||||
|
// A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a
|
||||||
|
// clean repo (objects just dangle harmlessly).
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
|
return 'off'
|
||||||
|
} finally {
|
||||||
|
core.endGroup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setupInner(settings: IGitSourceSettings): Promise<MirrorMode> {
|
||||||
|
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
|
||||||
|
const refKey = computeRefKey(settings)
|
||||||
|
plan = {mode: 'off', repoKey, refKey}
|
||||||
|
|
||||||
|
const lookup = await api.lookupRestore(repoKey, refKey)
|
||||||
|
if (lookup.kind === 'disabled') {
|
||||||
|
core.info('Mirror cache is disabled by the backend for this organization')
|
||||||
|
return 'off'
|
||||||
|
}
|
||||||
|
if (lookup.kind === 'error') {
|
||||||
|
core.info('Mirror cache backend unavailable; using standard checkout')
|
||||||
|
return 'off'
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lookup.kind === 'cold') {
|
||||||
|
const grant = await api.requestBaseUpload(repoKey)
|
||||||
|
if (grant.kind === 'grant') {
|
||||||
|
core.info(
|
||||||
|
'Cold repo: building the base mirror this run (full fetch, then upload)'
|
||||||
|
)
|
||||||
|
plan = {mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url}
|
||||||
|
return 'cold-build'
|
||||||
|
}
|
||||||
|
core.info(
|
||||||
|
'Cold repo: base build already in progress elsewhere; using standard checkout'
|
||||||
|
)
|
||||||
|
return 'off'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore: seed the base, then this branch's delta if present.
|
||||||
|
await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS)
|
||||||
|
// Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* +
|
||||||
|
// refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the
|
||||||
|
// delta fetch then advances the target ref to its current tip). A shallow upstream fetch
|
||||||
|
// (depth 1) populates only the target ref, so there we keep the base internal and let the
|
||||||
|
// delta fetch provide just that ref.
|
||||||
|
if (settings.fetchDepth <= 0) {
|
||||||
|
await exposeBaseRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
|
if (lookup.branch) {
|
||||||
|
try {
|
||||||
|
await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS)
|
||||||
|
} catch (error) {
|
||||||
|
core.info(`Branch delta seed skipped (${error}); base only`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
core.info(
|
||||||
|
`Seeded base mirror${
|
||||||
|
lookup.branch ? ' + branch delta' : ''
|
||||||
|
}; the GitHub fetch will be a tip delta`
|
||||||
|
)
|
||||||
|
plan = {mode: 'seeded', repoKey, refKey}
|
||||||
|
return 'seeded'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or
|
||||||
|
// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own
|
||||||
|
// try/catch for normal errors, plus scoped process guards (installed only for the upload
|
||||||
|
// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that
|
||||||
|
// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean
|
||||||
|
// exit(0). Safe because nothing checkout-relevant runs after this point.
|
||||||
|
export async function contribute(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const guard = (error: unknown): void => {
|
||||||
|
core.warning(
|
||||||
|
`WarpBuild mirror upload error ignored — checkout already complete: ${error}`
|
||||||
|
)
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
process.on('uncaughtException', guard)
|
||||||
|
process.on('unhandledRejection', guard)
|
||||||
|
try {
|
||||||
|
if (plan.mode === 'cold-build') {
|
||||||
|
await uploadBaseMirror(settings)
|
||||||
|
} else {
|
||||||
|
await uploadBranchDelta(settings)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
core.warning(`WarpBuild mirror upload skipped: ${error}`)
|
||||||
|
} finally {
|
||||||
|
// Seeded mode created the internal refs/wb/* namespace; strip it so the customer's
|
||||||
|
// .git carries no mirror fingerprint (matches upstream; won't break push --mirror).
|
||||||
|
if (plan.mode === 'seeded') {
|
||||||
|
await cleanupWbRefs(settings.repositoryPath)
|
||||||
|
}
|
||||||
|
process.removeListener('uncaughtException', guard)
|
||||||
|
process.removeListener('unhandledRejection', guard)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo
|
||||||
|
// (and count as "haves" for the delta negotiation). The bundle is a git-validated file;
|
||||||
|
// nothing is extracted into .git directly.
|
||||||
|
async function seedBundle(
|
||||||
|
repoPath: string,
|
||||||
|
url: string,
|
||||||
|
refNs: string
|
||||||
|
): Promise<void> {
|
||||||
|
const tmp = tempBundlePath('seed')
|
||||||
|
const what = refNs === BASE_REFNS ? 'base' : 'branch'
|
||||||
|
try {
|
||||||
|
await downloadTo(url, tmp, what)
|
||||||
|
await exec.exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'fetch',
|
||||||
|
'--quiet',
|
||||||
|
'--no-tags',
|
||||||
|
tmp,
|
||||||
|
`+refs/*:${refNs}/*`
|
||||||
|
])
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(tmp, {force: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy the seeded base's branches + tags out to the namespaces upstream uses
|
||||||
|
// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches
|
||||||
|
// upstream. Runs before the delta fetch, while the target ref is still absent, so the
|
||||||
|
// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe.
|
||||||
|
async function exposeBaseRefs(repoPath: string): Promise<void> {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
[
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'for-each-ref',
|
||||||
|
'--format=create refs/%(refname:lstrip=3) %(objectname)',
|
||||||
|
`${BASE_REFNS}/remotes/origin`,
|
||||||
|
`${BASE_REFNS}/tags`
|
||||||
|
],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
if (!out.trim()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so
|
||||||
|
// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort.
|
||||||
|
async function cleanupWbRefs(repoPath: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
if (!out.trim()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], {
|
||||||
|
input: Buffer.from(out)
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
core.debug(`WarpBuild ref cleanup skipped: ${error}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cold-build: after the forced full fetch the repo holds all branches under
|
||||||
|
// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would
|
||||||
|
// also sweep up the ephemeral triggering PR merge ref) as the base and upload.
|
||||||
|
async function uploadBaseMirror(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
if (!plan.baseUploadUrl) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const tmp = tempBundlePath('base')
|
||||||
|
try {
|
||||||
|
await exec.exec('git', [
|
||||||
|
'-C',
|
||||||
|
settings.repositoryPath,
|
||||||
|
'bundle',
|
||||||
|
'create',
|
||||||
|
tmp,
|
||||||
|
'--remotes=origin',
|
||||||
|
'--tags'
|
||||||
|
])
|
||||||
|
await httpPut(plan.baseUploadUrl, tmp)
|
||||||
|
core.info('Uploaded base mirror')
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(tmp, {force: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seeded: build the base-relative delta (target sha, excluding everything in the base)
|
||||||
|
// and overwrite this branch's bundle. Guarded by a per-branch lock server-side.
|
||||||
|
async function uploadBranchDelta(settings: IGitSourceSettings): Promise<void> {
|
||||||
|
if (!plan.refKey) {
|
||||||
|
return // tag / detached HEAD: nothing to roll
|
||||||
|
}
|
||||||
|
const repoPath = settings.repositoryPath
|
||||||
|
const sha = settings.commit
|
||||||
|
|
||||||
|
await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha])
|
||||||
|
try {
|
||||||
|
if (!(await hasBaseRefs(repoPath))) {
|
||||||
|
core.info('No base refs to diff against; branch delta skipped')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// If the tip is already in the base there is no delta to roll, and `git bundle
|
||||||
|
// create` errors on an empty range ("Refusing to create empty bundle"). Detect it
|
||||||
|
// first — and before taking the server lock, so an empty delta never blocks the
|
||||||
|
// other jobs racing the same branch.
|
||||||
|
if (await branchDeltaIsEmpty(repoPath)) {
|
||||||
|
core.info('Tip already contained in the base; no branch delta to upload')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha)
|
||||||
|
if (grant.kind !== 'grant') {
|
||||||
|
core.info(`Branch delta upload skipped (${grant.kind})`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const tmp = tempBundlePath('branch')
|
||||||
|
try {
|
||||||
|
// Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo
|
||||||
|
// has hundreds of base refs, and that many args overflows the Windows command-line
|
||||||
|
// limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the
|
||||||
|
// same base-relative (order-free) delta.
|
||||||
|
await exec.exec('git', [
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'bundle',
|
||||||
|
'create',
|
||||||
|
tmp,
|
||||||
|
UPLOAD_TIP_REF,
|
||||||
|
'--not',
|
||||||
|
`--glob=${BASE_REFNS}/*`
|
||||||
|
])
|
||||||
|
await httpPut(grant.url, tmp)
|
||||||
|
core.info(`Uploaded branch delta for '${plan.refKey}'`)
|
||||||
|
} finally {
|
||||||
|
await fs.promises.rm(tmp, {force: true})
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF],
|
||||||
|
{
|
||||||
|
ignoreReturnCode: true
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tip's delta against the base is empty when the tip is already reachable from the
|
||||||
|
// base — nothing new to bundle.
|
||||||
|
async function branchDeltaIsEmpty(repoPath: string): Promise<boolean> {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
[
|
||||||
|
'-C',
|
||||||
|
repoPath,
|
||||||
|
'rev-list',
|
||||||
|
'--count',
|
||||||
|
UPLOAD_TIP_REF,
|
||||||
|
'--not',
|
||||||
|
`--glob=${BASE_REFNS}/*`
|
||||||
|
],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
return out.trim() === '0'
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether any base ref was seeded — the guard that keeps the delta base-relative. The
|
||||||
|
// base is excluded by glob (see uploadBranchDelta), so we only need existence here.
|
||||||
|
async function hasBaseRefs(repoPath: string): Promise<boolean> {
|
||||||
|
let out = ''
|
||||||
|
await exec.exec(
|
||||||
|
'git',
|
||||||
|
['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS],
|
||||||
|
{silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}}
|
||||||
|
)
|
||||||
|
return out.trim().length > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle
|
||||||
|
// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling.
|
||||||
|
// Falls back to a single stream when the server doesn't support ranges.
|
||||||
|
async function downloadTo(
|
||||||
|
url: string,
|
||||||
|
dest: string,
|
||||||
|
what: string
|
||||||
|
): Promise<void> {
|
||||||
|
const started = Date.now()
|
||||||
|
// Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes.
|
||||||
|
const http = new HttpClient('warpbuild-checkout-mirror', undefined, {
|
||||||
|
socketTimeout: SEGMENT_TIMEOUT_MS,
|
||||||
|
keepAlive: true
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
const total = await probeRangeSize(http, url)
|
||||||
|
if (total === null) {
|
||||||
|
await downloadSingleStream(http, url, dest)
|
||||||
|
const bytes = (await fs.promises.stat(dest)).size
|
||||||
|
core.info(
|
||||||
|
`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const segments: [number, number][] = []
|
||||||
|
for (let off = 0; off < total; off += SEGMENT_SIZE) {
|
||||||
|
segments.push([off, Math.min(SEGMENT_SIZE, total - off)])
|
||||||
|
}
|
||||||
|
const width = Math.min(SEGMENT_CONCURRENCY, segments.length)
|
||||||
|
const fh = await fs.promises.open(dest, 'w')
|
||||||
|
try {
|
||||||
|
let next = 0
|
||||||
|
const worker = async (): Promise<void> => {
|
||||||
|
for (;;) {
|
||||||
|
const i = next++
|
||||||
|
if (i >= segments.length) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const [offset, count] = segments[i]
|
||||||
|
const buf = await downloadSegment(http, url, offset, count)
|
||||||
|
await fh.write(buf, 0, count, offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const pool: Promise<void>[] = []
|
||||||
|
for (let k = 0; k < width; k++) {
|
||||||
|
pool.push(worker())
|
||||||
|
}
|
||||||
|
await Promise.all(pool)
|
||||||
|
} finally {
|
||||||
|
await fh.close()
|
||||||
|
}
|
||||||
|
core.info(
|
||||||
|
`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`
|
||||||
|
)
|
||||||
|
} finally {
|
||||||
|
http.dispose()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size + elapsed/throughput for the mirror download o11y line.
|
||||||
|
function fmtSize(bytes: number): string {
|
||||||
|
return `${(bytes / 1e6).toFixed(1)} MB`
|
||||||
|
}
|
||||||
|
function fmtElapsed(startedMs: number, bytes: number): string {
|
||||||
|
const secs = (Date.now() - startedMs) / 1000
|
||||||
|
const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a'
|
||||||
|
return `${secs.toFixed(2)}s (${rate} MB/s)`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET bytes=0-0 to learn the total size and confirm range support; null → not supported.
|
||||||
|
async function probeRangeSize(
|
||||||
|
http: HttpClient,
|
||||||
|
url: string
|
||||||
|
): Promise<number | null> {
|
||||||
|
const res = await http.get(url, {Range: 'bytes=0-0'})
|
||||||
|
if (res.message.statusCode !== 206) {
|
||||||
|
res.message.destroy() // no range support; don't read a full 200 body into memory
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
await res.readBody() // 1 byte; return the socket to the keep-alive pool
|
||||||
|
const contentRange = res.message.headers['content-range']
|
||||||
|
const match =
|
||||||
|
typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null
|
||||||
|
const total = match ? parseInt(match[1], 10) : NaN
|
||||||
|
return Number.isFinite(total) && total > 0 ? total : null
|
||||||
|
}
|
||||||
|
|
||||||
|
// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes.
|
||||||
|
async function downloadSegment(
|
||||||
|
http: HttpClient,
|
||||||
|
url: string,
|
||||||
|
offset: number,
|
||||||
|
count: number
|
||||||
|
): Promise<Buffer> {
|
||||||
|
let lastErr: unknown
|
||||||
|
for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) {
|
||||||
|
try {
|
||||||
|
const res = await http.get(url, {
|
||||||
|
Range: `bytes=${offset}-${offset + count - 1}`
|
||||||
|
})
|
||||||
|
if (res.message.statusCode !== 206) {
|
||||||
|
await res.readBody() // drain before retrying so the socket is reusable
|
||||||
|
throw new Error(`range GET answered ${res.message.statusCode}`)
|
||||||
|
}
|
||||||
|
const buf = await res.readBodyBuffer?.()
|
||||||
|
if (!buf) {
|
||||||
|
throw new Error('http-client response lacks readBodyBuffer')
|
||||||
|
}
|
||||||
|
if (buf.length !== count) {
|
||||||
|
throw new Error(`short segment at ${offset}: ${buf.length}/${count}`)
|
||||||
|
}
|
||||||
|
return buf
|
||||||
|
} catch (error) {
|
||||||
|
lastErr = error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastErr instanceof Error
|
||||||
|
? lastErr
|
||||||
|
: new Error(`segment at ${offset} failed`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downloadSingleStream(
|
||||||
|
http: HttpClient,
|
||||||
|
url: string,
|
||||||
|
dest: string
|
||||||
|
): Promise<void> {
|
||||||
|
const res = await http.get(url)
|
||||||
|
const status = res.message.statusCode ?? 0
|
||||||
|
if (status < 200 || status >= 300) {
|
||||||
|
await res.readBody()
|
||||||
|
throw new Error(`download answered ${status}`)
|
||||||
|
}
|
||||||
|
await pipeline(res.message, fs.createWriteStream(dest))
|
||||||
|
}
|
||||||
|
|
||||||
|
async function httpPut(url: string, file: string): Promise<void> {
|
||||||
|
const stat = await fs.promises.stat(file)
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: 'PUT',
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body
|
||||||
|
body: fs.createReadStream(file) as any,
|
||||||
|
duplex: 'half',
|
||||||
|
headers: {'content-length': String(stat.size)},
|
||||||
|
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS)
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed
|
||||||
|
} as any)
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`PUT answered ${res.status}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a
|
||||||
|
// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file).
|
||||||
|
let mirrorTmpDir: string | undefined
|
||||||
|
function tempBundlePath(tag: string): string {
|
||||||
|
if (!mirrorTmpDir) {
|
||||||
|
mirrorTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wb-mirror-'))
|
||||||
|
}
|
||||||
|
return path.join(
|
||||||
|
mirrorTmpDir,
|
||||||
|
`wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kept for callers/tests: reset .git/objects to the empty `git init` shape.
|
||||||
|
export async function resetGitObjects(gitDir: string): Promise<void> {
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'objects'), {
|
||||||
|
recursive: true,
|
||||||
|
force: true
|
||||||
|
})
|
||||||
|
await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true})
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), {
|
||||||
|
recursive: true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`.
|
||||||
|
export async function hasGit(): Promise<boolean> {
|
||||||
|
return Boolean(await io.which('git', false))
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue