add lock awareness

README.md

@@ -1,12 +1,14 @@
 # WarpBuild Checkout
 
 This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
-replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's
-bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git
-alternates, so the fetch from GitHub downloads only the delta instead of the whole
-repository. On a cache miss, the run hydrates the mirror once (full clone + upload);
-mirrors expire on a server-configured TTL and re-hydrate.
+replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
+objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
+A later job checking out the same commit restores that snapshot and skips the fetch from
+GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
+matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
 
+- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS);
+  everything else runs exactly like upstream.
 - Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners.
 - Fail-open — any cache error degrades to stock `actions/checkout` behavior.
 - All fork code lives in `src/warpbuild/`

__test__/warpbuild-mirror.test.ts

@@ -6,8 +6,13 @@ import {
   afterEach,
   afterAll
 } from '@jest/globals'
+import * as exec from '@actions/exec'
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
 import {
   SKIP_NOT_WARPBUILD,
+  assertSafeTarMembers,
   computeDestinationRef,
   getMirrorCacheSkipReason
 } from '../src/warpbuild/mirror-cache.js'
@@ -131,6 +136,69 @@ describe('warpbuild snapshot cache', () => {
     })
   })
 
+  describe('assertSafeTarMembers', () => {
+    async function makeTar(
+      dir: string,
+      members: string[],
+      writeFiles = true
+    ): Promise<string> {
+      if (writeFiles) {
+        for (const m of members) {
+          const abs = path.join(dir, m)
+          await fs.promises.mkdir(path.dirname(abs), {recursive: true})
+          await fs.promises.writeFile(abs, 'x')
+        }
+      }
+      const tar = path.join(dir, 'out.tar')
+      await exec.exec('tar', ['-cf', tar, '-C', dir, ...members])
+      return tar
+    }
+
+    it('accepts objects/ and shallow members', async () => {
+      const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
+      try {
+        const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow'])
+        await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined()
+      } finally {
+        await fs.promises.rm(dir, {recursive: true, force: true})
+      }
+    })
+
+    it('rejects members outside objects/ and shallow (e.g. hooks)', async () => {
+      const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
+      try {
+        const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit'])
+        await expect(assertSafeTarMembers(tar)).rejects.toThrow(
+          /unexpected snapshot tar member/
+        )
+      } finally {
+        await fs.promises.rm(dir, {recursive: true, force: true})
+      }
+    })
+
+    it('rejects path traversal members', async () => {
+      const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
+      try {
+        await fs.promises.mkdir(path.join(dir, 'sub'))
+        const tar = path.join(dir, 'evil.tar')
+        // -P preserves the ../ member instead of stripping it.
+        await fs.promises.writeFile(path.join(dir, 'evil'), 'x')
+        await exec.exec('tar', [
+          '-cPf',
+          tar,
+          '-C',
+          path.join(dir, 'sub'),
+          '../evil'
+        ])
+        await expect(assertSafeTarMembers(tar)).rejects.toThrow(
+          /unexpected snapshot tar member/
+        )
+      } finally {
+        await fs.promises.rm(dir, {recursive: true, force: true})
+      }
+    })
+  })
+
   describe('computeDestinationRef', () => {
     it('maps the refs the fetch would have created', () => {
       expect(computeDestinationRef('refs/heads/main')).toBe(
@@ -172,6 +240,11 @@ describe('warpbuild snapshot cache', () => {
       expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
     })
 
+    it('maps 409 upload responses to locked (another job is uploading)', async () => {
+      stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
+      expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
+    })
+
     it('maps 403 to disabled (backend kill switch)', async () => {
       stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
       expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')

dist/index.js

@@ -41699,7 +41699,8 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
 
 // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
 // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
-// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
+// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
+// uploading this snapshot (skip upload); else = fall back.
 const API_TIMEOUT_MS = 10_000;
 function backend_api_baseUrl() {
     return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@@ -41753,6 +41754,10 @@ async function requestUploadURL(repoKey, sha) {
             core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
             return { kind: 'disabled' };
         }
+        if (res.status === 409) {
+            core_debug(`[wb-cache] upload-url answered 409 (locked)`);
+            return { kind: 'locked' };
+        }
         core_debug(`[wb-cache] upload-url answered ${res.status}`);
         return { kind: 'error' };
     }
@@ -41910,6 +41915,28 @@ async function contribute(settings) {
         endGroup();
     }
 }
+// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
+// component — the tar is remote and extracted into .git, so a crafted member could
+// escape (e.g. hooks/, ../) and run code during checkout.
+async function assertSafeTarMembers(tar) {
+    let listing = '';
+    await exec_exec('tar', ['-tf', tar], {
+        silent: true,
+        listeners: { stdout: (d) => (listing += d.toString()) }
+    });
+    for (const raw of listing.split('\n')) {
+        const member = raw.trim();
+        if (!member) {
+            continue;
+        }
+        const top = member.replace(/^\.\//, '').split('/')[0];
+        if (member.startsWith('/') ||
+            member.split('/').includes('..') ||
+            (top !== 'objects' && top !== 'shallow')) {
+            throw new Error(`unexpected snapshot tar member: ${member}`);
+        }
+    }
+}
 async function restoreSnapshot(settings, url, sha) {
     const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
     const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
@@ -41921,6 +41948,7 @@ async function restoreSnapshot(settings, url, sha) {
             throw new Error(`snapshot download answered ${res.status}`);
         }
         await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar));
+        await assertSafeTarMembers(tmpTar);
         await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]);
         const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true });
         if (check !== 0) {
@@ -41974,9 +42002,11 @@ async function uploadSnapshot(settings) {
         const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
         const upload = await requestUploadURL(repoKey, sha);
         if (upload.kind !== 'ok') {
-            info(upload.kind === 'disabled'
-                ? 'Snapshot cache is disabled by the backend; not uploading'
-                : 'Snapshot cache backend unavailable; not uploading');
+            info(upload.kind === 'locked'
+                ? 'Another job is already uploading this snapshot; skipping'
+                : upload.kind === 'disabled'
+                    ? 'Snapshot cache is disabled by the backend; not uploading'
+                    : 'Snapshot cache backend unavailable; not uploading');
             return;
         }
         const init = {

src/warpbuild/backend-api.ts

@@ -3,7 +3,8 @@ import * as core from '@actions/core'
 
 // Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
 // verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
-// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
+// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
+// uploading this snapshot (skip upload); else = fall back.
 
 const API_TIMEOUT_MS = 10_000
 
@@ -22,6 +23,7 @@ export type SnapshotLookup =
 export type UploadURLResult =
   | {kind: 'ok'; url: string}
   | {kind: 'disabled'}
+  | {kind: 'locked'}
   | {kind: 'error'}
 
 function baseUrl(): string {
@@ -89,6 +91,10 @@ export async function requestUploadURL(
       core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
       return {kind: 'disabled'}
     }
+    if (res.status === 409) {
+      core.debug(`[wb-cache] upload-url answered 409 (locked)`)
+      return {kind: 'locked'}
+    }
     core.debug(`[wb-cache] upload-url answered ${res.status}`)
     return {kind: 'error'}
   } catch (error) {

src/warpbuild/mirror-cache.ts

@@ -171,6 +171,31 @@ export async function contribute(settings: IGitSourceSettings): Promise<void> {
   }
 }
 
+// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
+// component — the tar is remote and extracted into .git, so a crafted member could
+// escape (e.g. hooks/, ../) and run code during checkout.
+export async function assertSafeTarMembers(tar: string): Promise<void> {
+  let listing = ''
+  await exec.exec('tar', ['-tf', tar], {
+    silent: true,
+    listeners: {stdout: (d: Buffer) => (listing += d.toString())}
+  })
+  for (const raw of listing.split('\n')) {
+    const member = raw.trim()
+    if (!member) {
+      continue
+    }
+    const top = member.replace(/^\.\//, '').split('/')[0]
+    if (
+      member.startsWith('/') ||
+      member.split('/').includes('..') ||
+      (top !== 'objects' && top !== 'shallow')
+    ) {
+      throw new Error(`unexpected snapshot tar member: ${member}`)
+    }
+  }
+}
+
 async function restoreSnapshot(
   settings: IGitSourceSettings,
   url: string,
@@ -189,6 +214,7 @@ async function restoreSnapshot(
       Readable.fromWeb(res.body as import('stream/web').ReadableStream),
       fs.createWriteStream(tmpTar)
     )
+    await assertSafeTarMembers(tmpTar)
     await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
 
     const check = await exec.exec(
@@ -253,9 +279,11 @@ async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
     const upload = await api.requestUploadURL(repoKey, sha)
     if (upload.kind !== 'ok') {
       core.info(
-        upload.kind === 'disabled'
-          ? 'Snapshot cache is disabled by the backend; not uploading'
-          : 'Snapshot cache backend unavailable; not uploading'
+        upload.kind === 'locked'
+          ? 'Another job is already uploading this snapshot; skipping'
+          : upload.kind === 'disabled'
+            ? 'Snapshot cache is disabled by the backend; not uploading'
+            : 'Snapshot cache backend unavailable; not uploading'
       )
       return
     }