diff --git a/README.md b/README.md index 5509e7d..d12db7f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ +# WarpBuild Checkout + +This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in +replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git` +objects a checkout produces are tarred and cached remotely, keyed by the exact commit SHA. +A later job checking out the same commit restores that snapshot and skips the fetch from +GitHub entirely — cutting request load (the main source of GitHub rate-limiting on +matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh. + +- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS); + everything else runs exactly like upstream. +- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners. +- Fail-open — any cache error degrades to stock `actions/checkout` behavior. +- All fork code lives in `src/warpbuild/` + +--- + [![Build and Test](https://github.com/actions/checkout/actions/workflows/test.yml/badge.svg)](https://github.com/actions/checkout/actions/workflows/test.yml) # Checkout v7 diff --git a/__test__/warpbuild-mirror.test.ts b/__test__/warpbuild-mirror.test.ts new file mode 100644 index 0000000..1d860f4 --- /dev/null +++ b/__test__/warpbuild-mirror.test.ts @@ -0,0 +1,273 @@ +import { + describe, + it, + expect, + beforeEach, + afterEach, + afterAll +} from '@jest/globals' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { + SKIP_NOT_WARPBUILD, + computeRefKey, + getMirrorCacheSkipReason, + resetGitObjects +} from '../src/warpbuild/mirror-cache.js' +import { + lookupRestore, + requestBaseUpload, + requestBranchUpload +} from '../src/warpbuild/backend-api.js' +import {IGitSourceSettings} from '../src/git-source-settings.js' + +const WB_ENV = [ + 'WARPBUILD_RUNNER_VERIFICATION_TOKEN', + 'WARPBUILD_HOST_URL', + 'GITHUB_REPOSITORY_ID', + 'GITHUB_REPOSITORY', + 'GITHUB_REF', + 'GITHUB_BASE_REF' +] +const savedEnv: {[key: string]: string | undefined} = {} +for (const key of WB_ENV) { + savedEnv[key] = process.env[key] +} + +const SHA = 'cd5255d20e23e050238affc045ba9beee35eaaf7' + +function settingsFor( + overrides: Partial = {} +): IGitSourceSettings { + return { + repositoryOwner: 'octocat', + repositoryName: 'hello-world', + repositoryPath: '/tmp/does-not-matter', + ref: 'refs/heads/main', + commit: SHA, + fetchDepth: 1, + fetchTags: false, + filter: undefined, + sparseCheckout: undefined, + lfs: false, + ...overrides + } as unknown as IGitSourceSettings +} + +function setWarpBuildEnv(): void { + process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] = 'test-token' + process.env['WARPBUILD_HOST_URL'] = 'https://api.example.dev' + process.env['GITHUB_REPOSITORY_ID'] = '123456789' + process.env['GITHUB_REPOSITORY'] = 'octocat/hello-world' + delete process.env['GITHUB_REF'] + delete process.env['GITHUB_BASE_REF'] +} + +describe('warpbuild mirror cache', () => { + beforeEach(() => { + setWarpBuildEnv() + }) + + afterAll(() => { + for (const key of WB_ENV) { + if (savedEnv[key] === undefined) { + delete process.env[key] + } else { + process.env[key] = savedEnv[key] + } + } + }) + + describe('getMirrorCacheSkipReason', () => { + it('returns null for the default checkout shape', () => { + expect(getMirrorCacheSkipReason(settingsFor())).toBeNull() + }) + + it('reports a non-WarpBuild runner', () => { + delete process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] + expect(getMirrorCacheSkipReason(settingsFor())).toBe(SKIP_NOT_WARPBUILD) + }) + + it('reports repository inputs that are not the workflow repo', () => { + expect( + getMirrorCacheSkipReason(settingsFor({repositoryName: 'other-repo'})) + ).toBe( + "repository 'octocat/other-repo' is not the workflow repository 'octocat/hello-world'" + ) + }) + + it('requires an exact commit sha', () => { + expect(getMirrorCacheSkipReason(settingsFor({commit: ''}))).toBe( + 'no exact commit sha to key on' + ) + expect(getMirrorCacheSkipReason(settingsFor({commit: 'main'}))).toBe( + 'no exact commit sha to key on' + ) + }) + + it('engages for depth 0/1, defers explicit shallow (>=2) to upstream', () => { + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 0}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 1}))).toBeNull() + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 2}))).toContain( + 'explicit shallow depth' + ) + expect(getMirrorCacheSkipReason(settingsFor({fetchDepth: 50}))).toContain( + 'explicit shallow depth' + ) + expect( + getMirrorCacheSkipReason(settingsFor({fetchTags: true})) + ).toBeNull() + }) + + it('skips filters and sparse checkouts', () => { + expect(getMirrorCacheSkipReason(settingsFor({filter: 'blob:none'}))).toBe( + 'a fetch filter is configured' + ) + expect( + getMirrorCacheSkipReason(settingsFor({sparseCheckout: ['src']})) + ).toBe('sparse checkout is configured') + }) + + it('engages for lfs (git objects from the mirror, lfs binaries fetched on top)', () => { + expect(getMirrorCacheSkipReason(settingsFor({lfs: true}))).toBeNull() + expect( + getMirrorCacheSkipReason(settingsFor({lfs: true, fetchDepth: 1})) + ).toBeNull() + }) + }) + + describe('computeRefKey', () => { + it('keys on the pushed branch (GITHUB_REF)', () => { + process.env['GITHUB_REF'] = 'refs/heads/feature/x' + expect(computeRefKey(settingsFor())).toBe('feature/x') + }) + + it('keys on the base branch for pull requests (merge sha is synthetic)', () => { + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + process.env['GITHUB_BASE_REF'] = 'main' + expect(computeRefKey(settingsFor())).toBe('main') + }) + + it('is empty for tags and bare pull refs (base only, no roll)', () => { + process.env['GITHUB_REF'] = 'refs/tags/v1.2.3' + expect(computeRefKey(settingsFor())).toBe('') + process.env['GITHUB_REF'] = 'refs/pull/42/merge' + expect(computeRefKey(settingsFor())).toBe('') + }) + + it('is empty for a detached sha checkout', () => { + expect(computeRefKey(settingsFor({ref: SHA}))).toBe('') + }) + + it('passes through an already-short branch name', () => { + expect(computeRefKey(settingsFor({ref: 'main'}))).toBe('main') + }) + }) + + describe('resetGitObjects', () => { + it('restores the empty git-init object layout, dropping restored junk', async () => { + const gitDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'wb-reset-') + ) + try { + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) + await fs.promises.writeFile( + path.join(gitDir, 'objects', 'ab'), + 'junk loose object' + ) + await fs.promises.writeFile(path.join(gitDir, 'shallow'), 'deadbeef\n') + + await resetGitObjects(gitDir) + + expect(fs.existsSync(path.join(gitDir, 'shallow'))).toBe(false) + expect(fs.existsSync(path.join(gitDir, 'objects', 'ab'))).toBe(false) + expect( + (await fs.promises.readdir(path.join(gitDir, 'objects'))).sort() + ).toEqual(['info', 'pack']) + } finally { + await fs.promises.rm(gitDir, {recursive: true, force: true}) + } + }) + }) + + describe('backend api http contract', () => { + const realFetch = globalThis.fetch + let lastUrl = '' + let lastInit: RequestInit | undefined + + afterEach(() => { + globalThis.fetch = realFetch + }) + + function stubFetch(status: number, body: unknown): void { + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + lastUrl = String(input) + lastInit = init + return new Response(JSON.stringify(body), {status}) + }) as typeof fetch + } + + it('lookupRestore maps 200 to a restore plan keyed by ref', async () => { + stubFetch(200, { + base: {url: 'https://s3/base', size_bytes: 42}, + branch: {url: 'https://s3/branch'} + }) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.base.url).toBe('https://s3/base') + expect(result.branch?.url).toBe('https://s3/branch') + } + expect(lastUrl).toContain('ref=main') + }) + + it('lookupRestore maps a null branch (base but no delta yet)', async () => { + stubFetch(200, {base: {url: 'https://s3/base'}, branch: null}) + const result = await lookupRestore('123', 'main') + expect(result.kind).toBe('restore') + if (result.kind === 'restore') { + expect(result.branch).toBeNull() + } + }) + + it('lookupRestore maps 404 to cold, 403 to disabled, 500 to error', async () => { + stubFetch(404, {sub_code: 'NFE_GITMIRROR'}) + expect((await lookupRestore('123', 'main')).kind).toBe('cold') + stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'}) + expect((await lookupRestore('123', 'main')).kind).toBe('disabled') + stubFetch(500, {}) + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('lookupRestore maps network failure to error', async () => { + globalThis.fetch = (async () => { + throw new Error('boom') + }) as typeof fetch + expect((await lookupRestore('123', 'main')).kind).toBe('error') + }) + + it('requestBaseUpload maps 200 to a grant, 409 to locked', async () => { + stubFetch(200, {url: 'https://s3/put-base'}) + const grant = await requestBaseUpload('123') + expect(grant.kind).toBe('grant') + if (grant.kind === 'grant') { + expect(grant.url).toBe('https://s3/put-base') + } + expect(lastInit?.method).toBe('POST') + stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'}) + expect((await requestBaseUpload('123')).kind).toBe('locked') + }) + + it('requestBranchUpload maps 200 to a grant, 403 to disabled', async () => { + stubFetch(200, {url: 'https://s3/put-branch'}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe('grant') + stubFetch(403, {}) + expect((await requestBranchUpload('123', 'main', SHA)).kind).toBe( + 'disabled' + ) + }) + }) +}) diff --git a/action.yml b/action.yml index 5b0524f..32c44db 100644 --- a/action.yml +++ b/action.yml @@ -112,6 +112,8 @@ outputs: description: 'The branch, tag or SHA that was checked out' commit: description: 'The commit SHA that was checked out' + cache-hit: + description: 'Whether the checkout was restored from the WarpBuild snapshot cache (true/false)' runs: using: node24 main: dist/index.js diff --git a/dist/index.js b/dist/index.js index b381bd2..5c58625 100644 --- a/dist/index.js +++ b/dist/index.js @@ -31883,6 +31883,64 @@ function qstring(str) { /******/ } /******/ /************************************************************************/ +/******/ /* webpack/runtime/create fake namespace object */ +/******/ (() => { +/******/ var getProto = Object.getPrototypeOf ? (obj) => (Object.getPrototypeOf(obj)) : (obj) => (obj.__proto__); +/******/ var leafPrototypes; +/******/ // create a fake namespace object +/******/ // mode & 1: value is a module id, require it +/******/ // mode & 2: merge all properties of value into the ns +/******/ // mode & 4: return value when already ns object +/******/ // mode & 16: return value when it's Promise-like +/******/ // mode & 8|1: behave like require +/******/ __nccwpck_require__.t = function(value, mode) { +/******/ if(mode & 1) value = this(value); +/******/ if(mode & 8) return value; +/******/ if(typeof value === 'object' && value) { +/******/ if((mode & 4) && value.__esModule) return value; +/******/ if((mode & 16) && typeof value.then === 'function') return value; +/******/ } +/******/ var ns = Object.create(null); +/******/ __nccwpck_require__.r(ns); +/******/ var def = {}; +/******/ leafPrototypes = leafPrototypes || [null, getProto({}), getProto([]), getProto(getProto)]; +/******/ for(var current = mode & 2 && value; typeof current == 'object' && !~leafPrototypes.indexOf(current); current = getProto(current)) { +/******/ Object.getOwnPropertyNames(current).forEach((key) => (def[key] = () => (value[key]))); +/******/ } +/******/ def['default'] = () => (value); +/******/ __nccwpck_require__.d(ns, def); +/******/ return ns; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __nccwpck_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __nccwpck_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ /******/ /* webpack/runtime/compat */ /******/ /******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/"; @@ -32063,8 +32121,10 @@ function file_command_prepareKeyValueMessage(key, value) { const external_path_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("path"); // EXTERNAL MODULE: external "http" var external_http_ = __nccwpck_require__(8611); +var external_http_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_http_, 2); // EXTERNAL MODULE: external "https" var external_https_ = __nccwpck_require__(5692); +var external_https_namespaceObject = /*#__PURE__*/__nccwpck_require__.t(external_https_, 2); ;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/proxy.js function getProxyUrl(reqUrl) { const usingSsl = reqUrl.protocol === 'https:'; @@ -32157,7 +32217,7 @@ class DecodedURL extends URL { } //# sourceMappingURL=proxy.js.map // EXTERNAL MODULE: ./node_modules/tunnel/index.js -var node_modules_tunnel = __nccwpck_require__(770); +var tunnel = __nccwpck_require__(770); // EXTERNAL MODULE: ./node_modules/undici/index.js var undici = __nccwpck_require__(6752); ;// CONCATENATED MODULE: ./node_modules/@actions/http-client/lib/index.js @@ -32235,7 +32295,7 @@ const HttpResponseRetryCodes = [ HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout ]; -const RetryableHttpVerbs = (/* unused pure expression or super */ null && (['OPTIONS', 'GET', 'DELETE', 'HEAD'])); +const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; class HttpClientError extends Error { @@ -32584,7 +32644,7 @@ class lib_HttpClient { } getAgentDispatcher(serverUrl) { const parsedUrl = new URL(serverUrl); - const proxyUrl = pm.getProxyUrl(parsedUrl); + const proxyUrl = getProxyUrl(parsedUrl); const useProxy = proxyUrl && proxyUrl.hostname; if (!useProxy) { return; @@ -32595,7 +32655,7 @@ class lib_HttpClient { const info = {}; info.parsedUrl = requestUrl; const usingSsl = info.parsedUrl.protocol === 'https:'; - info.httpModule = usingSsl ? https : http; + info.httpModule = usingSsl ? external_https_namespaceObject : external_http_namespaceObject; const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; @@ -32694,7 +32754,7 @@ class lib_HttpClient { } _getAgent(parsedUrl) { let agent; - const proxyUrl = pm.getProxyUrl(parsedUrl); + const proxyUrl = getProxyUrl(parsedUrl); const useProxy = proxyUrl && proxyUrl.hostname; if (this._keepAlive && useProxy) { agent = this._proxyAgent; @@ -32709,7 +32769,7 @@ class lib_HttpClient { const usingSsl = parsedUrl.protocol === 'https:'; let maxSockets = 100; if (this.requestOptions) { - maxSockets = this.requestOptions.maxSockets || http.globalAgent.maxSockets; + maxSockets = this.requestOptions.maxSockets || external_http_.globalAgent.maxSockets; } // This is `useProxy` again, but we need to check `proxyURl` directly for TypeScripts's flow analysis. if (proxyUrl && proxyUrl.hostname) { @@ -32734,7 +32794,7 @@ class lib_HttpClient { // if tunneling agent isn't assigned create a new agent if (!agent) { const options = { keepAlive: this._keepAlive, maxSockets }; - agent = usingSsl ? new https.Agent(options) : new http.Agent(options); + agent = usingSsl ? new external_https_.Agent(options) : new external_http_.Agent(options); this._agent = agent; } if (usingSsl && this._ignoreSslError) { @@ -32757,7 +32817,7 @@ class lib_HttpClient { return proxyAgent; } const usingSsl = parsedUrl.protocol === 'https:'; - proxyAgent = new ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && { + proxyAgent = new undici.ProxyAgent(Object.assign({ uri: proxyUrl.href, pipelining: !this._keepAlive ? 0 : 1 }, ((proxyUrl.username || proxyUrl.password) && { token: `Basic ${Buffer.from(`${proxyUrl.username}:${proxyUrl.password}`).toString('base64')}` }))); this._proxyAgentDispatcher = proxyAgent; @@ -41692,6 +41752,617 @@ function ref_helper_select(obj, path) { return ref_helper_select(obj[key], path.substr(i + 1)); } +;// CONCATENATED MODULE: external "stream/promises" +const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("stream/promises"); +;// CONCATENATED MODULE: ./src/warpbuild/backend-api.ts +/* eslint-disable i18n-text/no-en -- upstream convention */ + +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. +const API_TIMEOUT_MS = 10_000; +function backend_api_baseUrl() { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, ''); +} +function authHeader() { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}`; +} +function backend_api_endpoint(p) { + return `${backend_api_baseUrl()}/api/v1/git-mirrors/${p}`; +} +async function lookupRestore(repoKey, ref) { + try { + const res = await fetch(`${backend_api_endpoint('restore-url')}?repo_key=${encodeURIComponent(repoKey)}&ref=${encodeURIComponent(ref)}`, { + headers: { authorization: authHeader() }, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const body = (await res.json()); + return { kind: 'restore', base: body.base, branch: body.branch ?? null }; + } + if (res.status === 404) { + return { kind: 'cold' }; + } + if (res.status === 403) { + core_debug('[wb-cache] restore-url answered 403 (disabled)'); + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] restore-url answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] restore-url failed: ${error}`); + return { kind: 'error' }; + } +} +async function requestUpload(p, body) { + try { + const res = await fetch(backend_api_endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }); + if (res.status === 200) { + const b = (await res.json()); + return { kind: 'grant', url: b.url }; + } + if (res.status === 409) { + return { kind: 'locked' }; + } + if (res.status === 403) { + return { kind: 'disabled' }; + } + core_debug(`[wb-cache] ${p} answered ${res.status}`); + return { kind: 'error' }; + } + catch (error) { + core_debug(`[wb-cache] ${p} failed: ${error}`); + return { kind: 'error' }; + } +} +// Cold repo: request the single-flight grant to build + upload the base mirror. +async function requestBaseUpload(repoKey) { + return requestUpload('base/upload-url', { repo_key: repoKey }); +} +// Warm repo: request the grant to overwrite this branch's delta bundle. +async function requestBranchUpload(repoKey, ref, sha) { + return requestUpload('branch/upload-url', { repo_key: repoKey, ref, sha }); +} + +;// CONCATENATED MODULE: ./src/warpbuild/mirror-cache.ts +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ + + + + + + + + + + +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. +const UPLOAD_TIMEOUT_MS = 15 * 60_000; +// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a +// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a +// WAN); parallel range GETs reach link rate (~200 MB/s). +const SEGMENT_SIZE = 4 * 1024 * 1024; +const SEGMENT_CONCURRENCY = 8; +const SEGMENT_TIMEOUT_MS = 30_000; +const SEGMENT_RETRIES = 5; +// Logged at info when off; the WARPBUILD_* env is only present on our runners. +const SKIP_NOT_WARPBUILD = 'not running on a WarpBuild runner (WARPBUILD_* env not present)'; +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/; +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base'; +const BRANCH_REFNS = 'refs/wb/branch'; +const UPLOAD_TIP_REF = 'refs/wb/tip'; +let plan = { mode: 'off', repoKey: '', refKey: '' }; +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. +function getMirrorCacheSkipReason(settings) { + if (!process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL']) { + return SKIP_NOT_WARPBUILD; + } + if (!process.env['GITHUB_REPOSITORY_ID']) { + return 'GITHUB_REPOSITORY_ID is not set'; + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}`; + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'`; + } + const server = (settings.githubServerUrl || 'https://github.com').replace(/\/+$/, ''); + if (server !== 'https://github.com') { + return `server '${server}' is not github.com`; + } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on'; + } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout`; + } + if (settings.filter) { + return 'a fetch filter is configured'; + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured'; + } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. + return null; +} +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +function computeRefKey(settings) { + const baseRef = process.env['GITHUB_BASE_REF']; // set on pull_request events + if (baseRef) { + return baseRef; + } + const ref = process.env['GITHUB_REF'] || settings.ref || ''; + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length); + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { + return ''; + } + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref; +} +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). +async function setup(settings) { + const mode = await setupImpl(settings); + setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false'); + return mode; +} +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +async function abandon(settings) { + plan = { mode: 'off', repoKey: '', refKey: '' }; + setOutput('cache-hit', 'false'); + await cleanupWbRefs(settings.repositoryPath); +} +async function setupImpl(settings) { + plan = { mode: 'off', repoKey: '', refKey: '' }; + const skipReason = getMirrorCacheSkipReason(settings); + if (skipReason) { + info(`WarpBuild mirror cache skipped: ${skipReason}`); + return 'off'; + } + startGroup('WarpBuild: checkout mirror cache'); + try { + return await setupInner(settings); + } + catch (error) { + warning(`WarpBuild mirror cache unavailable, using standard checkout: ${error}`); + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath); + return 'off'; + } + finally { + endGroup(); + } +} +async function setupInner(settings) { + const repoKey = process.env['GITHUB_REPOSITORY_ID']; + const refKey = computeRefKey(settings); + plan = { mode: 'off', repoKey, refKey }; + const lookup = await lookupRestore(repoKey, refKey); + if (lookup.kind === 'disabled') { + info('Mirror cache is disabled by the backend for this organization'); + return 'off'; + } + if (lookup.kind === 'error') { + info('Mirror cache backend unavailable; using standard checkout'); + return 'off'; + } + if (lookup.kind === 'cold') { + const grant = await requestBaseUpload(repoKey); + if (grant.kind === 'grant') { + info('Cold repo: building the base mirror this run (full fetch, then upload)'); + plan = { mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url }; + return 'cold-build'; + } + info('Cold repo: base build already in progress elsewhere; using standard checkout'); + return 'off'; + } + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS); + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath); + } + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS); + } + catch (error) { + info(`Branch delta seed skipped (${error}); base only`); + } + } + info(`Seeded base mirror${lookup.branch ? ' + branch delta' : ''}; the GitHub fetch will be a tip delta`); + plan = { mode: 'seeded', repoKey, refKey }; + return 'seeded'; +} +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. +async function contribute(settings) { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return; + } + const guard = (error) => { + warning(`WarpBuild mirror upload error ignored — checkout already complete: ${error}`); + process.exit(0); + }; + process.on('uncaughtException', guard); + process.on('unhandledRejection', guard); + try { + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings); + } + else { + await uploadBranchDelta(settings); + } + } + catch (error) { + warning(`WarpBuild mirror upload skipped: ${error}`); + } + finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath); + } + process.removeListener('uncaughtException', guard); + process.removeListener('unhandledRejection', guard); + } +} +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle(repoPath, url, refNs) { + const tmp = tempBundlePath('seed'); + const what = refNs === BASE_REFNS ? 'base' : 'branch'; + try { + await downloadTo(url, tmp, what); + await exec_exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } +} +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath) { + let out = ''; + await exec_exec('git', [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); +} +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath) { + try { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + if (!out.trim()) { + return; + } + await exec_exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }); + } + catch (error) { + core_debug(`WarpBuild ref cleanup skipped: ${error}`); + } +} +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings) { + if (!plan.baseUploadUrl) { + return; + } + const tmp = tempBundlePath('base'); + try { + await exec_exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]); + await httpPut(plan.baseUploadUrl, tmp); + info('Uploaded base mirror'); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } +} +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings) { + if (!plan.refKey) { + return; // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath; + const sha = settings.commit; + await exec_exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]); + try { + if (!(await hasBaseRefs(repoPath))) { + info('No base refs to diff against; branch delta skipped'); + return; + } + // If the tip is already in the base there is no delta to roll, and `git bundle + // create` errors on an empty range ("Refusing to create empty bundle"). Detect it + // first — and before taking the server lock, so an empty delta never blocks the + // other jobs racing the same branch. + if (await branchDeltaIsEmpty(repoPath)) { + info('Tip already contained in the base; no branch delta to upload'); + return; + } + const grant = await requestBranchUpload(plan.repoKey, plan.refKey, sha); + if (grant.kind !== 'grant') { + info(`Branch delta upload skipped (${grant.kind})`); + return; + } + const tmp = tempBundlePath('branch'); + try { + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. + await exec_exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ]); + await httpPut(grant.url, tmp); + info(`Uploaded branch delta for '${plan.refKey}'`); + } + finally { + await external_fs_namespaceObject.promises.rm(tmp, { force: true }); + } + } + finally { + await exec_exec('git', ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], { + ignoreReturnCode: true + }); + } +} +// The tip's delta against the base is empty when the tip is already reachable from the +// base — nothing new to bundle. +async function branchDeltaIsEmpty(repoPath) { + let out = ''; + await exec_exec('git', [ + '-C', + repoPath, + 'rev-list', + '--count', + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out.trim() === '0'; +} +// Whether any base ref was seeded — the guard that keeps the delta base-relative. The +// base is excluded by glob (see uploadBranchDelta), so we only need existence here. +async function hasBaseRefs(repoPath) { + let out = ''; + await exec_exec('git', ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], { silent: true, listeners: { stdout: (d) => (out += d.toString()) } }); + return out.trim().length > 0; +} +// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle +// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. +// Falls back to a single stream when the server doesn't support ranges. +async function downloadTo(url, dest, what) { + const started = Date.now(); + // Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes. + const http = new lib_HttpClient('warpbuild-checkout-mirror', undefined, { + socketTimeout: SEGMENT_TIMEOUT_MS, + keepAlive: true + }); + try { + const total = await probeRangeSize(http, url); + if (total === null) { + await downloadSingleStream(http, url, dest); + const bytes = (await external_fs_namespaceObject.promises.stat(dest)).size; + info(`Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}`); + return; + } + const segments = []; + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]); + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length); + const fh = await external_fs_namespaceObject.promises.open(dest, 'w'); + try { + let next = 0; + const worker = async () => { + for (;;) { + const i = next++; + if (i >= segments.length) { + return; + } + const [offset, count] = segments[i]; + const buf = await downloadSegment(http, url, offset, count); + await fh.write(buf, 0, count, offset); + } + }; + const pool = []; + for (let k = 0; k < width; k++) { + pool.push(worker()); + } + await Promise.all(pool); + } + finally { + await fh.close(); + } + info(`Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}`); + } + finally { + http.dispose(); + } +} +// Size + elapsed/throughput for the mirror download o11y line. +function fmtSize(bytes) { + return `${(bytes / 1e6).toFixed(1)} MB`; +} +function fmtElapsed(startedMs, bytes) { + const secs = (Date.now() - startedMs) / 1000; + const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a'; + return `${secs.toFixed(2)}s (${rate} MB/s)`; +} +// GET bytes=0-0 to learn the total size and confirm range support; null → not supported. +async function probeRangeSize(http, url) { + const res = await http.get(url, { Range: 'bytes=0-0' }); + if (res.message.statusCode !== 206) { + res.message.destroy(); // no range support; don't read a full 200 body into memory + return null; + } + await res.readBody(); // 1 byte; return the socket to the keep-alive pool + const contentRange = res.message.headers['content-range']; + const match = typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null; + const total = match ? parseInt(match[1], 10) : NaN; + return Number.isFinite(total) && total > 0 ? total : null; +} +// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes. +async function downloadSegment(http, url, offset, count) { + let lastErr; + for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { + try { + const res = await http.get(url, { + Range: `bytes=${offset}-${offset + count - 1}` + }); + if (res.message.statusCode !== 206) { + await res.readBody(); // drain before retrying so the socket is reusable + throw new Error(`range GET answered ${res.message.statusCode}`); + } + const buf = await res.readBodyBuffer?.(); + if (!buf) { + throw new Error('http-client response lacks readBodyBuffer'); + } + if (buf.length !== count) { + throw new Error(`short segment at ${offset}: ${buf.length}/${count}`); + } + return buf; + } + catch (error) { + lastErr = error; + } + } + throw lastErr instanceof Error + ? lastErr + : new Error(`segment at ${offset} failed`); +} +async function downloadSingleStream(http, url, dest) { + const res = await http.get(url); + const status = res.message.statusCode ?? 0; + if (status < 200 || status >= 300) { + await res.readBody(); + throw new Error(`download answered ${status}`); + } + await (0,promises_namespaceObject.pipeline)(res.message, external_fs_namespaceObject.createWriteStream(dest)); +} +async function httpPut(url, file) { + const stat = await external_fs_namespaceObject.promises.stat(file); + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: external_fs_namespaceObject.createReadStream(file), + duplex: 'half', + headers: { 'content-length': String(stat.size) }, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + }); + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`); + } +} +// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a +// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file). +let mirrorTmpDir; +function tempBundlePath(tag) { + if (!mirrorTmpDir) { + mirrorTmpDir = external_fs_namespaceObject.mkdtempSync(external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), 'wb-mirror-')); + } + return external_path_namespaceObject.join(mirrorTmpDir, `wb-${tag}-${external_crypto_namespaceObject.randomBytes(12).toString('hex')}.bundle`); +} +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. +async function resetGitObjects(gitDir) { + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }); + await fs.promises.rm(path.join(gitDir, 'shallow'), { force: true }); + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }); + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }); +} +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +async function hasGit() { + return Boolean(await io.which('git', false)); +} + ;// CONCATENATED MODULE: ./src/git-source-provider.ts @@ -41705,6 +42376,7 @@ function ref_helper_select(obj, path) { + async function getSource(settings) { // Repository URL info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`); @@ -41724,6 +42396,7 @@ async function getSource(settings) { const git = await getGitCommandManager(settings); endGroup(); let authHelper = null; + let warpbuildMode = 'off'; try { if (git) { authHelper = createAuthHelper(git, settings); @@ -41774,6 +42447,8 @@ async function getSource(settings) { await git.init(objectFormat); await git.remoteAdd('origin', repositoryUrl); endGroup(); + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildMode = await setup(settings); } // Disable automatic garbage collection startGroup('Disabling automatic garbage collection'); @@ -41813,7 +42488,30 @@ async function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } - if (settings.fetchDepth <= 0) { + let warpbuildFetchDone = false; + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = getRefSpec(settings.ref, settings.commit); + await git.fetch(refSpec, fetchOptions); + if (!(await testRef(git, settings.ref, settings.commit))) { + throw new Error(`The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.`); + } + warpbuildFetchDone = true; + } + catch (error) { + warning(`WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}`); + await abandon(settings); + warpbuildMode = 'off'; + } + } + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. + } + else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = getRefSpecForAllHistory(settings.ref, settings.commit); await git.fetch(refSpec, fetchOptions); @@ -41905,6 +42603,16 @@ async function getSource(settings) { setOutput('commit', commitSHA.trim()); // Check for incorrect pull request merge commit await checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.githubServerUrl); + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await contribute(settings); + } + catch (error) { + warning(`WarpBuild mirror upload skipped: ${error}`); + } } finally { // Remove auth diff --git a/package.json b/package.json index 4a38df9..15f1741 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@actions/core": "^3.0.1", "@actions/exec": "^3.0.0", "@actions/github": "^9.1.1", + "@actions/http-client": "^4.0.1", "@actions/io": "^3.0.2", "@actions/tool-cache": "^4.0.0" }, diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index b9c1d35..df892aa 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -9,6 +9,7 @@ import * as path from 'path' import * as refHelper from './ref-helper.js' import * as stateHelper from './state-helper.js' import * as urlHelper from './url-helper.js' +import * as warpbuildMirror from './warpbuild/mirror-cache.js' import { MinimumGitSparseCheckoutVersion, IGitCommandManager @@ -40,6 +41,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.endGroup() let authHelper: gitAuthHelper.IGitAuthHelper | null = null + let warpbuildMode: warpbuildMirror.MirrorMode = 'off' try { if (git) { authHelper = gitAuthHelper.createAuthHelper(git, settings) @@ -130,6 +132,9 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.init(objectFormat) await git.remoteAdd('origin', repositoryUrl) core.endGroup() + + // WarpBuild snapshot cache: hit = objects restored, fetch below is skipped + warpbuildMode = await warpbuildMirror.setup(settings) } // Disable automatic garbage collection @@ -185,7 +190,33 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } - if (settings.fetchDepth <= 0) { + let warpbuildFetchDone = false + if (warpbuildMode === 'seeded') { + // Seeded from the mirror: fetch only the tip delta, negotiating against the seeded + // objects. No depth — the base is full history, so this stays non-shallow. If it + // fails for any reason, disengage cleanly and fall through to the standard fetch. + try { + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(refSpec, fetchOptions) + if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { + throw new Error( + `The ref '${settings.ref}' does not point to the expected commit '${settings.commit}'. ` + + `The ref may have been updated after the workflow was triggered.` + ) + } + warpbuildFetchDone = true + } catch (error) { + core.warning( + `WarpBuild mirror delta fetch failed; falling back to standard checkout: ${error}` + ) + await warpbuildMirror.abandon(settings) + warpbuildMode = 'off' + } + } + + if (warpbuildFetchDone) { + // The seeded delta fetch already brought in the target commit. + } else if (warpbuildMode === 'cold-build' || settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, @@ -313,6 +344,16 @@ export async function getSource(settings: IGitSourceSettings): Promise { settings.commit, settings.githubServerUrl ) + + // WarpBuild mirror: best-effort upload, run LAST in getSource so it can never affect + // the checkout result. Belt-and-suspenders around contribute()'s own try/catch and the + // scoped process guard it installs (which ignores any uncaught upload error now that + // the checkout is already complete). + try { + await warpbuildMirror.contribute(settings) + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } } finally { // Remove auth if (authHelper) { diff --git a/src/warpbuild/backend-api.ts b/src/warpbuild/backend-api.ts new file mode 100644 index 0000000..b052abc --- /dev/null +++ b/src/warpbuild/backend-api.ts @@ -0,0 +1,120 @@ +/* eslint-disable i18n-text/no-en -- upstream convention */ +import * as core from '@actions/core' + +// Client for backend-core's git-mirror endpoints, authed by the runner verification +// token. The action seeds from a bare base mirror + a per-branch delta bundle (both +// presigned GETs), delta-fetches the tip from GitHub, then uploads the refreshed +// per-branch bundle — or, on a cold repo, the base — via a presigned PUT. Every call +// fails closed to stock checkout. + +const API_TIMEOUT_MS = 10_000 + +export interface Presigned { + url: string + size_bytes?: number +} + +// GET restore-url: what to seed before the delta fetch. 'cold' = no base yet. +export type RestoreLookup = + | {kind: 'restore'; base: Presigned; branch: Presigned | null} + | {kind: 'cold'} + | {kind: 'disabled'} + | {kind: 'error'} + +// POST *upload-url: a presigned PUT, or a signal to skip (locked/disabled). +export type UploadGrant = + | {kind: 'grant'; url: string} + | {kind: 'locked'} + | {kind: 'disabled'} + | {kind: 'error'} + +function baseUrl(): string { + return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '') +} + +function authHeader(): string { + return `Bearer ${process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || ''}` +} + +function endpoint(p: string): string { + return `${baseUrl()}/api/v1/git-mirrors/${p}` +} + +export async function lookupRestore( + repoKey: string, + ref: string +): Promise { + try { + const res = await fetch( + `${endpoint('restore-url')}?repo_key=${encodeURIComponent( + repoKey + )}&ref=${encodeURIComponent(ref)}`, + { + headers: {authorization: authHeader()}, + signal: AbortSignal.timeout(API_TIMEOUT_MS) + } + ) + if (res.status === 200) { + const body = (await res.json()) as { + base: Presigned + branch: Presigned | null + } + return {kind: 'restore', base: body.base, branch: body.branch ?? null} + } + if (res.status === 404) { + return {kind: 'cold'} + } + if (res.status === 403) { + core.debug('[wb-cache] restore-url answered 403 (disabled)') + return {kind: 'disabled'} + } + core.debug(`[wb-cache] restore-url answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] restore-url failed: ${error}`) + return {kind: 'error'} + } +} + +async function requestUpload(p: string, body: object): Promise { + try { + const res = await fetch(endpoint(p), { + method: 'POST', + headers: { + authorization: authHeader(), + 'content-type': 'application/json' + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(API_TIMEOUT_MS) + }) + if (res.status === 200) { + const b = (await res.json()) as {url: string} + return {kind: 'grant', url: b.url} + } + if (res.status === 409) { + return {kind: 'locked'} + } + if (res.status === 403) { + return {kind: 'disabled'} + } + core.debug(`[wb-cache] ${p} answered ${res.status}`) + return {kind: 'error'} + } catch (error) { + core.debug(`[wb-cache] ${p} failed: ${error}`) + return {kind: 'error'} + } +} + +// Cold repo: request the single-flight grant to build + upload the base mirror. +export async function requestBaseUpload(repoKey: string): Promise { + return requestUpload('base/upload-url', {repo_key: repoKey}) +} + +// Warm repo: request the grant to overwrite this branch's delta bundle. +export async function requestBranchUpload( + repoKey: string, + ref: string, + sha: string +): Promise { + return requestUpload('branch/upload-url', {repo_key: repoKey, ref, sha}) +} diff --git a/src/warpbuild/mirror-cache.ts b/src/warpbuild/mirror-cache.ts new file mode 100644 index 0000000..8d2953b --- /dev/null +++ b/src/warpbuild/mirror-cache.ts @@ -0,0 +1,625 @@ +/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */ +import * as core from '@actions/core' +import * as exec from '@actions/exec' +import * as io from '@actions/io' +import {HttpClient} from '@actions/http-client' +import * as crypto from 'crypto' +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {pipeline} from 'stream/promises' +import {IGitSourceSettings} from '../git-source-settings.js' +import * as api from './backend-api.js' + +// WarpBuild checkout cache: a bare base mirror + per-branch orderless delta bundles on +// S3. We seed the base (+ this branch's delta) before the fetch, so the GitHub fetch is +// only the tip delta — or nothing. Each run overwrites its branch's bundle (base-relative, +// so order-free); a cold repo single-flights the base build. Fail-open: any error → +// standard checkout. + +const UPLOAD_TIMEOUT_MS = 15 * 60_000 + +// Concurrent range-download tuning, matched to WarpBuilds/cache: 4 MiB blocks, 8-wide, over a +// keep-alive http.Agent. A single stream saturates at the per-connection ceiling (~15 MB/s on a +// WAN); parallel range GETs reach link rate (~200 MB/s). +const SEGMENT_SIZE = 4 * 1024 * 1024 +const SEGMENT_CONCURRENCY = 8 +const SEGMENT_TIMEOUT_MS = 30_000 +const SEGMENT_RETRIES = 5 + +// Logged at info when off; the WARPBUILD_* env is only present on our runners. +export const SKIP_NOT_WARPBUILD = + 'not running on a WarpBuild runner (WARPBUILD_* env not present)' + +const SHA_PATTERN = /^[0-9a-f]{40}([0-9a-f]{24})?$/ + +// Seeded bundles land here, out of the user's ref space; excluded objects for deltas. +const BASE_REFNS = 'refs/wb/base' +const BRANCH_REFNS = 'refs/wb/branch' +const UPLOAD_TIP_REF = 'refs/wb/tip' + +export type MirrorMode = 'off' | 'seeded' | 'cold-build' + +interface Plan { + mode: MirrorMode + repoKey: string + refKey: string + baseUploadUrl?: string // cold-build only +} +let plan: Plan = {mode: 'off', repoKey: '', refKey: ''} + +// Null = attempt the cache; else a reason to log. We engage only where the result matches +// upstream: full history (fetch-depth 0) or the default depth 1. An explicit shallow depth +// (>= 2) is a deliberate request we can't honour (the mirror is full history), so we defer +// to upstream. sparse/filter change the object set we model, so they defer too. LFS does +// NOT defer: the bundle carries the git objects (including LFS pointer blobs), and the +// stock `git lfs fetch`/`checkout` steps pull the actual LFS binaries from GitHub on top, +// exactly as upstream — the mirror only accelerates the git-object half. +export function getMirrorCacheSkipReason( + settings: IGitSourceSettings +): string | null { + if ( + !process.env['WARPBUILD_RUNNER_VERIFICATION_TOKEN'] || + !process.env['WARPBUILD_HOST_URL'] + ) { + return SKIP_NOT_WARPBUILD + } + if (!process.env['GITHUB_REPOSITORY_ID']) { + return 'GITHUB_REPOSITORY_ID is not set' + } + const checkoutRepo = `${settings.repositoryOwner}/${settings.repositoryName}` + if (checkoutRepo !== process.env['GITHUB_REPOSITORY']) { + return `repository '${checkoutRepo}' is not the workflow repository '${process.env['GITHUB_REPOSITORY']}'` + } + const server = (settings.githubServerUrl || 'https://github.com').replace( + /\/+$/, + '' + ) + if (server !== 'https://github.com') { + return `server '${server}' is not github.com` + } + if (!settings.commit || !SHA_PATTERN.test(settings.commit)) { + return 'no exact commit sha to key on' + } + // Respect an explicit shallow request: depth 0 (all) and the default depth 1 engage the + // mirror; anything deeper is a deliberate shallow the mirror can't reproduce → upstream. + if (settings.fetchDepth > 1) { + return `fetch-depth ${settings.fetchDepth} is an explicit shallow depth; using upstream checkout` + } + if (settings.filter) { + return 'a fetch filter is configured' + } + if (settings.sparseCheckout) { + return 'sparse checkout is configured' + } + // LFS intentionally does not skip — see the note above getMirrorCacheSkipReason. + return null +} + +// The durable branch to key the per-branch bundle on. For pull_request events the merge +// SHA is synthetic, so we key on the base branch; otherwise the pushed branch. '' for +// tags / detached HEAD → seed the base only, upload nothing. +export function computeRefKey(settings: IGitSourceSettings): string { + const baseRef = process.env['GITHUB_BASE_REF'] // set on pull_request events + if (baseRef) { + return baseRef + } + const ref = process.env['GITHUB_REF'] || settings.ref || '' + if (ref.startsWith('refs/heads/')) { + return ref.substring('refs/heads/'.length) + } + if (ref.startsWith('refs/tags/') || ref.startsWith('refs/pull/')) { + return '' + } + // Already a short branch name, or empty. + return SHA_PATTERN.test(ref) ? '' : ref +} + +// Runs after `git init`, before the fetch. Returns the mode the fetch/contribute steps +// branch on. Never throws. Sets the `cache-hit` output (true only when seeded from cache). +export async function setup(settings: IGitSourceSettings): Promise { + const mode = await setupImpl(settings) + core.setOutput('cache-hit', mode === 'seeded' ? 'true' : 'false') + return mode +} + +// Called by getSource when a mirror-seeded fetch fails: clear our refs and disengage, so +// the caller can retry the standard fetch against a clean repo. Also re-sets cache-hit to +// false since we're no longer serving from cache. +export async function abandon(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} + core.setOutput('cache-hit', 'false') + await cleanupWbRefs(settings.repositoryPath) +} + +async function setupImpl(settings: IGitSourceSettings): Promise { + plan = {mode: 'off', repoKey: '', refKey: ''} + const skipReason = getMirrorCacheSkipReason(settings) + if (skipReason) { + core.info(`WarpBuild mirror cache skipped: ${skipReason}`) + return 'off' + } + core.startGroup('WarpBuild: checkout mirror cache') + try { + return await setupInner(settings) + } catch (error) { + core.warning( + `WarpBuild mirror cache unavailable, using standard checkout: ${error}` + ) + // A partial seed may have left refs/wb/*; strip it so the stock fetch runs against a + // clean repo (objects just dangle harmlessly). + await cleanupWbRefs(settings.repositoryPath) + return 'off' + } finally { + core.endGroup() + } +} + +async function setupInner(settings: IGitSourceSettings): Promise { + const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string + const refKey = computeRefKey(settings) + plan = {mode: 'off', repoKey, refKey} + + const lookup = await api.lookupRestore(repoKey, refKey) + if (lookup.kind === 'disabled') { + core.info('Mirror cache is disabled by the backend for this organization') + return 'off' + } + if (lookup.kind === 'error') { + core.info('Mirror cache backend unavailable; using standard checkout') + return 'off' + } + + if (lookup.kind === 'cold') { + const grant = await api.requestBaseUpload(repoKey) + if (grant.kind === 'grant') { + core.info( + 'Cold repo: building the base mirror this run (full fetch, then upload)' + ) + plan = {mode: 'cold-build', repoKey, refKey, baseUploadUrl: grant.url} + return 'cold-build' + } + core.info( + 'Cold repo: base build already in progress elsewhere; using standard checkout' + ) + return 'off' + } + + // Restore: seed the base, then this branch's delta if present. + await seedBundle(settings.repositoryPath, lookup.base.url, BASE_REFNS) + // Full-history checkout: mirror the base's branches + tags into refs/remotes/origin/* + + // refs/tags/* so the resulting .git matches an upstream `fetch-depth: 0` checkout (the + // delta fetch then advances the target ref to its current tip). A shallow upstream fetch + // (depth 1) populates only the target ref, so there we keep the base internal and let the + // delta fetch provide just that ref. + if (settings.fetchDepth <= 0) { + await exposeBaseRefs(settings.repositoryPath) + } + if (lookup.branch) { + try { + await seedBundle(settings.repositoryPath, lookup.branch.url, BRANCH_REFNS) + } catch (error) { + core.info(`Branch delta seed skipped (${error}); base only`) + } + } + core.info( + `Seeded base mirror${ + lookup.branch ? ' + branch delta' : '' + }; the GitHub fetch will be a tip delta` + ) + plan = {mode: 'seeded', repoKey, refKey} + return 'seeded' +} + +// Runs LAST in getSource — the checkout has already succeeded. Uploads the base (cold) or +// this branch's refreshed delta (seeded). Hardened so it can NEVER fail the step: its own +// try/catch for normal errors, plus scoped process guards (installed only for the upload +// window) that turn an *uncaught* error from the git/upload work — e.g. a spawn error that +// bypasses the promise chain, like the arg-length crash we hit on Windows — into a clean +// exit(0). Safe because nothing checkout-relevant runs after this point. +export async function contribute(settings: IGitSourceSettings): Promise { + if (plan.mode !== 'cold-build' && plan.mode !== 'seeded') { + return + } + const guard = (error: unknown): void => { + core.warning( + `WarpBuild mirror upload error ignored — checkout already complete: ${error}` + ) + process.exit(0) + } + process.on('uncaughtException', guard) + process.on('unhandledRejection', guard) + try { + if (plan.mode === 'cold-build') { + await uploadBaseMirror(settings) + } else { + await uploadBranchDelta(settings) + } + } catch (error) { + core.warning(`WarpBuild mirror upload skipped: ${error}`) + } finally { + // Seeded mode created the internal refs/wb/* namespace; strip it so the customer's + // .git carries no mirror fingerprint (matches upstream; won't break push --mirror). + if (plan.mode === 'seeded') { + await cleanupWbRefs(settings.repositoryPath) + } + process.removeListener('uncaughtException', guard) + process.removeListener('unhandledRejection', guard) + } +} + +// Download a bundle and fetch its refs into refNs/* so its objects seed the local repo +// (and count as "haves" for the delta negotiation). The bundle is a git-validated file; +// nothing is extracted into .git directly. +async function seedBundle( + repoPath: string, + url: string, + refNs: string +): Promise { + const tmp = tempBundlePath('seed') + const what = refNs === BASE_REFNS ? 'base' : 'branch' + try { + await downloadTo(url, tmp, what) + await exec.exec('git', [ + '-C', + repoPath, + 'fetch', + '--quiet', + '--no-tags', + tmp, + `+refs/*:${refNs}/*` + ]) + } finally { + await fs.promises.rm(tmp, {force: true}) + } +} + +// Copy the seeded base's branches + tags out to the namespaces upstream uses +// (refs/remotes/origin/* + refs/tags/*), so a full-history checkout's .git matches +// upstream. Runs before the delta fetch, while the target ref is still absent, so the +// `create` directives don't collide. update-ref --stdin keeps it Windows arg-length safe. +async function exposeBaseRefs(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + [ + '-C', + repoPath, + 'for-each-ref', + '--format=create refs/%(refname:lstrip=3) %(objectname)', + `${BASE_REFNS}/remotes/origin`, + `${BASE_REFNS}/tags` + ], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) +} + +// Remove the internal refs/wb/* namespace. Objects stay (reachable from the real refs), so +// the checkout is unaffected; only the mirror's bookkeeping refs go. Best-effort. +async function cleanupWbRefs(repoPath: string): Promise { + try { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--format=delete %(refname)', 'refs/wb'], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + if (!out.trim()) { + return + } + await exec.exec('git', ['-C', repoPath, 'update-ref', '--stdin'], { + input: Buffer.from(out) + }) + } catch (error) { + core.debug(`WarpBuild ref cleanup skipped: ${error}`) + } +} + +// Cold-build: after the forced full fetch the repo holds all branches under +// refs/remotes/origin/* and all tags. Bundle exactly those (not `--all`, which would +// also sweep up the ephemeral triggering PR merge ref) as the base and upload. +async function uploadBaseMirror(settings: IGitSourceSettings): Promise { + if (!plan.baseUploadUrl) { + return + } + const tmp = tempBundlePath('base') + try { + await exec.exec('git', [ + '-C', + settings.repositoryPath, + 'bundle', + 'create', + tmp, + '--remotes=origin', + '--tags' + ]) + await httpPut(plan.baseUploadUrl, tmp) + core.info('Uploaded base mirror') + } finally { + await fs.promises.rm(tmp, {force: true}) + } +} + +// Seeded: build the base-relative delta (target sha, excluding everything in the base) +// and overwrite this branch's bundle. Guarded by a per-branch lock server-side. +async function uploadBranchDelta(settings: IGitSourceSettings): Promise { + if (!plan.refKey) { + return // tag / detached HEAD: nothing to roll + } + const repoPath = settings.repositoryPath + const sha = settings.commit + + await exec.exec('git', ['-C', repoPath, 'update-ref', UPLOAD_TIP_REF, sha]) + try { + if (!(await hasBaseRefs(repoPath))) { + core.info('No base refs to diff against; branch delta skipped') + return + } + // If the tip is already in the base there is no delta to roll, and `git bundle + // create` errors on an empty range ("Refusing to create empty bundle"). Detect it + // first — and before taking the server lock, so an empty delta never blocks the + // other jobs racing the same branch. + if (await branchDeltaIsEmpty(repoPath)) { + core.info('Tip already contained in the base; no branch delta to upload') + return + } + + const grant = await api.requestBranchUpload(plan.repoKey, plan.refKey, sha) + if (grant.kind !== 'grant') { + core.info(`Branch delta upload skipped (${grant.kind})`) + return + } + + const tmp = tempBundlePath('branch') + try { + // Exclude the base with a single --glob, not one `^ref` arg per ref: a large repo + // has hundreds of base refs, and that many args overflows the Windows command-line + // limit (ENAMETOOLONG). The glob matches the multi-level seeded refs and yields the + // same base-relative (order-free) delta. + await exec.exec('git', [ + '-C', + repoPath, + 'bundle', + 'create', + tmp, + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ]) + await httpPut(grant.url, tmp) + core.info(`Uploaded branch delta for '${plan.refKey}'`) + } finally { + await fs.promises.rm(tmp, {force: true}) + } + } finally { + await exec.exec( + 'git', + ['-C', repoPath, 'update-ref', '-d', UPLOAD_TIP_REF], + { + ignoreReturnCode: true + } + ) + } +} + +// The tip's delta against the base is empty when the tip is already reachable from the +// base — nothing new to bundle. +async function branchDeltaIsEmpty(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + [ + '-C', + repoPath, + 'rev-list', + '--count', + UPLOAD_TIP_REF, + '--not', + `--glob=${BASE_REFNS}/*` + ], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + return out.trim() === '0' +} + +// Whether any base ref was seeded — the guard that keeps the delta base-relative. The +// base is excluded by glob (see uploadBranchDelta), so we only need existence here. +async function hasBaseRefs(repoPath: string): Promise { + let out = '' + await exec.exec( + 'git', + ['-C', repoPath, 'for-each-ref', '--count=1', '--format=1', BASE_REFNS], + {silent: true, listeners: {stdout: (d: Buffer) => (out += d.toString())}} + ) + return out.trim().length > 0 +} + +// Download a presigned GET to dest with concurrent HTTP range requests, so the base bundle +// (tens/hundreds of MiB) transfers at link rate instead of the single-stream ~15 MB/s ceiling. +// Falls back to a single stream when the server doesn't support ranges. +async function downloadTo( + url: string, + dest: string, + what: string +): Promise { + const started = Date.now() + // Keep-alive http.Agent (like WarpBuilds/cache) — real parallel sockets; bare fetch under-parallelizes. + const http = new HttpClient('warpbuild-checkout-mirror', undefined, { + socketTimeout: SEGMENT_TIMEOUT_MS, + keepAlive: true + }) + try { + const total = await probeRangeSize(http, url) + if (total === null) { + await downloadSingleStream(http, url, dest) + const bytes = (await fs.promises.stat(dest)).size + core.info( + `Mirror download [${what}]: ${fmtSize(bytes)} single-stream, no range support, ${fmtElapsed(started, bytes)}` + ) + return + } + const segments: [number, number][] = [] + for (let off = 0; off < total; off += SEGMENT_SIZE) { + segments.push([off, Math.min(SEGMENT_SIZE, total - off)]) + } + const width = Math.min(SEGMENT_CONCURRENCY, segments.length) + const fh = await fs.promises.open(dest, 'w') + try { + let next = 0 + const worker = async (): Promise => { + for (;;) { + const i = next++ + if (i >= segments.length) { + return + } + const [offset, count] = segments[i] + const buf = await downloadSegment(http, url, offset, count) + await fh.write(buf, 0, count, offset) + } + } + const pool: Promise[] = [] + for (let k = 0; k < width; k++) { + pool.push(worker()) + } + await Promise.all(pool) + } finally { + await fh.close() + } + core.info( + `Mirror download [${what}]: ${fmtSize(total)} in ${segments.length} ranged segments ×${width}, ${fmtElapsed(started, total)}` + ) + } finally { + http.dispose() + } +} + +// Size + elapsed/throughput for the mirror download o11y line. +function fmtSize(bytes: number): string { + return `${(bytes / 1e6).toFixed(1)} MB` +} +function fmtElapsed(startedMs: number, bytes: number): string { + const secs = (Date.now() - startedMs) / 1000 + const rate = secs > 0 ? (bytes / 1e6 / secs).toFixed(1) : 'n/a' + return `${secs.toFixed(2)}s (${rate} MB/s)` +} + +// GET bytes=0-0 to learn the total size and confirm range support; null → not supported. +async function probeRangeSize( + http: HttpClient, + url: string +): Promise { + const res = await http.get(url, {Range: 'bytes=0-0'}) + if (res.message.statusCode !== 206) { + res.message.destroy() // no range support; don't read a full 200 body into memory + return null + } + await res.readBody() // 1 byte; return the socket to the keep-alive pool + const contentRange = res.message.headers['content-range'] + const match = + typeof contentRange === 'string' ? contentRange.match(/\/(\d+)\s*$/) : null + const total = match ? parseInt(match[1], 10) : NaN + return Number.isFinite(total) && total > 0 ? total : null +} + +// One range GET with retries over the shared keep-alive client; returns exactly `count` bytes. +async function downloadSegment( + http: HttpClient, + url: string, + offset: number, + count: number +): Promise { + let lastErr: unknown + for (let attempt = 0; attempt <= SEGMENT_RETRIES; attempt++) { + try { + const res = await http.get(url, { + Range: `bytes=${offset}-${offset + count - 1}` + }) + if (res.message.statusCode !== 206) { + await res.readBody() // drain before retrying so the socket is reusable + throw new Error(`range GET answered ${res.message.statusCode}`) + } + const buf = await res.readBodyBuffer?.() + if (!buf) { + throw new Error('http-client response lacks readBodyBuffer') + } + if (buf.length !== count) { + throw new Error(`short segment at ${offset}: ${buf.length}/${count}`) + } + return buf + } catch (error) { + lastErr = error + } + } + throw lastErr instanceof Error + ? lastErr + : new Error(`segment at ${offset} failed`) +} + +async function downloadSingleStream( + http: HttpClient, + url: string, + dest: string +): Promise { + const res = await http.get(url) + const status = res.message.statusCode ?? 0 + if (status < 200 || status >= 300) { + await res.readBody() + throw new Error(`download answered ${status}`) + } + await pipeline(res.message, fs.createWriteStream(dest)) +} + +async function httpPut(url: string, file: string): Promise { + const stat = await fs.promises.stat(file) + const res = await fetch(url, { + method: 'PUT', + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- node stream body + body: fs.createReadStream(file) as any, + duplex: 'half', + headers: {'content-length': String(stat.size)}, + signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- duplex not yet typed + } as any) + if (!res.ok) { + throw new Error(`PUT answered ${res.status}`) + } +} + +// Bundles live in a private mkdtemp'd dir (0700) — a co-tenant can't pre-empt a write via a +// planted symlink, and mkdtempSync is the primitive CodeQL accepts (js/insecure-temporary-file). +let mirrorTmpDir: string | undefined +function tempBundlePath(tag: string): string { + if (!mirrorTmpDir) { + mirrorTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'wb-mirror-')) + } + return path.join( + mirrorTmpDir, + `wb-${tag}-${crypto.randomBytes(12).toString('hex')}.bundle` + ) +} + +// Kept for callers/tests: reset .git/objects to the empty `git init` shape. +export async function resetGitObjects(gitDir: string): Promise { + await fs.promises.rm(path.join(gitDir, 'objects'), { + recursive: true, + force: true + }) + await fs.promises.rm(path.join(gitDir, 'shallow'), {force: true}) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'info'), { + recursive: true + }) + await fs.promises.mkdir(path.join(gitDir, 'objects', 'pack'), { + recursive: true + }) +} + +// Kept so tests referencing io stay valid; the mirror path no longer shells to `tar`. +export async function hasGit(): Promise { + return Boolean(await io.which('git', false)) +}