GitHubivarvong

ply

main
ply6 git calls28 ms25 KiB5 cachedprotocol v2
270 lines17 KiBRaw
1/* oxlint-disable import/max-dependencies -- result.js is the 0.2.0 return type; this file was already at the module cap */
2// Node only (node:fs, node:path): checkoutTree's files written into a directory, laid out as `git clone` lays them out.
3// Not part of any Workers bundle — the Workers-safe core is checkout.js, which yields the files as streams.
4//
5// writeFiles(dir, files, { limits, platform }) → Result<{ files, bytes, symlinks, gitlinks }> files: checkoutTree's `files`
6// checkoutDir(url, { dir, ref, path }, opts) → Result of checkoutTree's record without `files`, plus `written`
7// writeFiles takes both fields. checkoutDir takes `dir` and the tracer; the rest default (ref: the remote's HEAD, path:
8// the whole tree, batch: CHECKOUT_BATCH, limits: DEFAULT_LIMITS, platform: this process's, the others none).
9//
10// Layout as git's checkout (entry.c): a 100755 file is created 0777 & ~umask, any other 0666 & ~umask; a 120000 is a
11// symlink whose target is the blob, never followed; a 160000 gitlink is an empty directory (an uninitialised submodule).
12//
13// Path safety. `dir` must be absent or empty, so everything under it is ours. Every path goes through validatePath (no
14// `..`, `.`, empty or absolute component, no .git in any spelling git refuses) and git's verify_path for this platform
15// (validate.js checkoutPathRefusal): on Windows a name with a backslash (`..\x` would leave `dir`), a colon (an NTFS
16// stream), a reserved device name or a trailing dot or space is `unsupported`, as git for Windows refuses it. Each
17// directory on the way is made one level at a time and lstat'd: anything but a real directory there (a symlink this
18// checkout made, or a file — the same name on a case-insensitive filesystem counts) is `bad_request` (what: 'checkout
19// path'), so nothing is ever written through a link. Files are opened O_CREAT|O_EXCL ('wx'), which neither follows a symlink nor overwrites; symlink()
20// never replaces. A stream that fails (a blob that does not hash to its oid) removes its file before the error is
21// returned as Err; a checkout that returns Err leaves `dir` partial. A filesystem error that is not EILSEQ is
22// `unsupported` (kind: filesystem). checkoutTree's Err is checkoutDir's.
23import { mkdir, readdir, lstat, open, symlink, unlink } from 'node:fs/promises';
24// oxlint-disable-next-line unicorn/import-style -- `path` names a checkout path throughout this file
25import { join } from 'node:path';
26import { platform as processPlatform } from 'node:process';
27import { fail, isPlyError, plyError } from './errors.js';
28import type { ErrorOf, PlyError } from './errors.js';
29import { err, getOrElse, mapOption, ok } from './result.js';
30import type { Err, Option, Result } from './result.js';
31import { DEFAULT_LIMITS, limitError, parseLimits } from './limits.js';
32import type { LimitsInput } from './limits.js';
33import { validatePath, checkoutPathRefusal } from './validate.js';
34import { concat, encodeBytes } from './bytes.js';
35import { MODE_EXEC, MODE_LINK, MODE_COMMIT } from './format.js';
36import { checkoutTree } from './checkout.js';
37import type { CheckoutFile, CheckoutResult } from './checkout.js';
38import type { ReadOptions } from './read.js';
39import { thrownCodeIs } from './boundary.js';
40import type { HttpAuth } from './http.js';
41import type { Tracer, Span } from './otel.js';
42import type { Quirks, Transport } from './transport.js';
43import type { Input } from './absent.js';
44import { field, withDefaults } from './absent.js';
46/* oxlint-disable no-await-in-loop -- files are written one at a time, in order, each stream drained before the next */
47/* oxlint-disable unicorn/explicit-length-check, require-unicode-regexp -- bundle budget: the JS's truthy length checks and non-unicode regexes, byte for byte (no behaviour change) */
49/* oxlint-disable no-magic-numbers -- the permission bits git creates files with (entry.c), before the umask, and checkout.ts BATCH */
50const FILE_MODE = 0o666;
51const EXEC_MODE = 0o777;
52// The batch checkoutTree uses when its own parameter is omitted. checkoutDir takes it as a required field.
53export const CHECKOUT_BATCH = 1000;
54/* oxlint-enable no-magic-numbers */
56// A tree name is bytes (#43): one that is not UTF-8 arrives with escaped bytes (bytes.js decodeBytes) and is written as
57// those bytes through a Buffer path, as git writes it. A filesystem that refuses the bytes (APFS: EILSEQ) is a typed
58// refusal naming the path, never a file under a U+FFFD name.
59const fsPath = (dir: string, rel: string): Result<string | Buffer, PlyError> => {
60 const p = join(dir, rel);
61 if (p.isWellFormed()) return ok(p);
62 const bytes = encodeBytes(p, 'checkout path');
63 return bytes.ok ? ok(Buffer.from(bytes.value)) : bytes;
64};
65// A wrapper: the call site is the decision point (`refused(path, e)`), so the body stays the `fail()` itself.
66const refused = (path: string, e: unknown) =>
67 (thrownCodeIs(e, 'EILSEQ') ? fail('unsupported', `checkout: the filesystem refuses the name bytes of ${JSON.stringify(path)}`, { what: 'checkout path', path, cause: e }) : e);
69const collision = (path: string, why: string): Err<ErrorOf<'bad_request'>> => fail('bad_request', `checkout: ${path} ${why}`, { what: 'checkout path', path });
71type WithKind<E, K extends string> = E & { readonly kind: K };
72const kinded = <E extends PlyError, K extends string>(error: E, kind: K): Err<WithKind<E, K>> => err(Object.assign(error, { kind }));
73const isResultErr = (v: unknown): v is Err<PlyError> => {
74 if (typeof v !== 'object' || v === null || !('ok' in v) || v.ok !== false || !('error' in v)) return false;
75 return isPlyError(v.error);
76};
77// EILSEQ is name_bytes. Any other filesystem error is unsupported, kind filesystem (one decision point).
78const nameBytes = (named: unknown): Err<WithKind<PlyError, 'name_bytes'>> | null =>
79 (isResultErr(named) ? kinded(named.error, 'name_bytes') : null);
80const filesystem = (path: string, e: unknown): Err<WithKind<ErrorOf<'unsupported'>, 'filesystem'>> =>
81 kinded(plyError('unsupported', `checkout: the filesystem refused ${JSON.stringify(path)}`, { what: 'checkout path', path, cause: e }), 'filesystem');
82const caught = (e: unknown, path: string): Err<WriteFilesError> => (isPlyError(e) ? err(e) : filesystem(path, e));
84// The refusals writeFiles itself builds. A PlyError from validatePath, encodeBytes, parseLimits, or a blob stream
85// is returned as that error (its own code and miss); it is not re-tagged.
86export type WriteFilesError =
87 | WithKind<ErrorOf<'bad_request'>, 'not_empty'>
88 | WithKind<ErrorOf<'bad_request'>, 'collision'>
89 | WithKind<ErrorOf<'unsupported'>, 'name_bytes'>
90 | WithKind<ErrorOf<'unsupported'>, 'path_refused'>
91 | WithKind<ErrorOf<'limit_exceeded'>, 'symlink_target'>
92 | WithKind<ErrorOf<'unsupported'>, 'filesystem'>
93 | PlyError;
95// Effect: `dir` exists and is empty afterwards, or not_empty.
96async function emptyDir(dir: string): Promise<Result<true, WithKind<ErrorOf<'bad_request'>, 'not_empty'> | WithKind<ErrorOf<'unsupported'>, 'filesystem'>>> {
97 try {
98 await mkdir(dir, { recursive: true });
99 if ((await readdir(dir)).length) return kinded(fail('bad_request', `checkout: ${dir} is not empty`, { what: 'dir' }).error, 'not_empty');
100 } catch (e) { return filesystem(dir, e); }
101 return ok(true);
104// Effect: read a stream into one buffer, at most `max` bytes (a symlink target). A stream that errors (a blob that does
105// not hash to its oid) is that error as an Err, like a regular file's.
106async function slurp(stream: ReadableStream<Uint8Array>, max: number, path: string): Promise<Result<Uint8Array<ArrayBuffer>, WriteFilesError>> {
107 const parts: Uint8Array[] = []; let n = 0;
108 try {
109 for await (const c of stream) { n += c.length; if (n > max) return kinded(limitError('symlink target', n, max), 'symlink_target'); parts.push(c); }
110 } catch (e) { return caught(e, path); }
111 return ok(concat(parts));
114// What writeFiles wrote: regular files (and their bytes), symlinks, and gitlinks (as empty directories).
115export type Written = { files: number; bytes: number; symlinks: number; gitlinks: number };
116async function ensureDir(dir: string, rel: string, made: Set<string>): Promise<Result<true, WriteFilesError>> {
117 if (made.has(rel)) return ok(true);
118 const i = rel.lastIndexOf('/');
119 const up = await ensureDir(dir, i < 0 ? '' : rel.slice(0, i), made);
120 if (!up.ok) return up;
121 const named = fsPath(dir, rel);
122 if (!named.ok) return named;
123 const target = named.value;
124 const created = await mkdir(target).then(() => ok(true), (e: unknown): Result<true, WriteFilesError> => {
125 if (thrownCodeIs(e, 'EEXIST')) return ok(true);
126 const bytes = nameBytes(refused(rel, e));
127 if (bytes) return bytes;
128 return filesystem(rel, e);
129 });
130 if (!created.ok) return created;
131 const st = await lstat(target).then(s => ok(s), (e: unknown) => caught(e, rel));
132 if (!st.ok) return st;
133 if (!st.value.isDirectory()) return kinded(collision(rel, 'is already a file or symlink here; refusing to write through it').error, 'collision');
134 made.add(rel);
135 return ok(true);
138// limits: any subset (a key left out is the default), parsed once before anything is written.
139export type WriteFilesOptions = { readonly limits: LimitsInput; readonly platform: string };
140// Complexity is the refusal branches. Pulling them into a helper would rename the decision-point function
141// (the id is `writeFiles`, and the registry matches that name).
142// oxlint-disable-next-line eslint/complexity -- see above
143export async function writeFiles(dir: string, files: AsyncIterable<CheckoutFile>, options: WriteFilesOptions): Promise<Result<Written, WriteFilesError>> {
144 const { platform } = options;
145 const checked = parseLimits(options.limits);
146 if (!checked.ok) return checked;
147 const limits = checked.value;
148 const maxObject = limits.maxObjectSize;
149 const ready = await emptyDir(dir);
150 if (!ready.ok) return ready;
151 const made = new Set(['']);
152 const out = { files: 0, bytes: 0, symlinks: 0, gitlinks: 0 };
153 // Pull by hand: a blob that fails before the entry is yielded rejects next(), and that rejection is an Err.
154 // for-await would nest the body one level deeper than the depth cap.
155 const iter = files[Symbol.asyncIterator]();
156 const stop = async (r: Err<WriteFilesError>): Promise<Err<WriteFilesError>> => { if (iter.return) await iter.return(); return r; };
157 for (;;) {
158 let next: IteratorResult<CheckoutFile>;
159 try { next = await iter.next(); }
160 catch (e) { return caught(e, dir); }
161 if (next.done === true) break;
162 const f = next.value;
163 const valid = validatePath(f.path, limits);
164 if (!valid.ok) return stop(valid);
165 const refusal = checkoutPathRefusal(f.path, f.mode, platform);
166 if (!refusal.ok) return stop(refusal);
167 const why = getOrElse<string | null>(refusal.value, null);
168 if (why !== null) {
169 // a gitlink has no stream; every other entry does (CheckoutFile)
170 if (f.mode !== MODE_COMMIT) await f.stream.cancel().catch(() => { /* refusing: the stream is not read */ });
171 return stop(kinded(fail('unsupported', `checkout: git refuses to check out ${JSON.stringify(f.path)} on ${platform}: ${why}`, { what: 'checkout path', path: f.path }).error, 'path_refused'));
172 }
173 const named = fsPath(dir, f.path);
174 if (!named.ok) return stop(named);
175 const full = named.value;
176 const slash = f.path.lastIndexOf('/');
177 const parent = await ensureDir(dir, slash < 0 ? '' : f.path.slice(0, slash), made);
178 if (!parent.ok) return stop(parent);
179 if (f.mode === MODE_COMMIT) { const sub = await ensureDir(dir, f.path, made); if (!sub.ok) return stop(sub); out.gitlinks++; continue; }
180 // a regular file or a symlink: checkoutTree streams every entry that is not a gitlink (CheckoutFile)
181 const stream = f.stream;
182 if (f.mode === MODE_LINK) {
183 const target = await slurp(stream, maxObject, f.path);
184 if (!target.ok) return stop(target);
185 const linked = await symlink(Buffer.from(target.value.buffer, target.value.byteOffset, target.value.length), full).then(() => ok(true), (e: unknown): Result<true, WriteFilesError> => {
186 if (thrownCodeIs(e, 'EEXIST')) return kinded(collision(f.path, 'already exists').error, 'collision');
187 const bytes = nameBytes(refused(f.path, e));
188 if (bytes) return bytes;
189 return filesystem(f.path, e);
190 });
191 if (!linked.ok) return stop(linked);
192 out.symlinks++; continue;
193 }
194 const opened = await open(full, 'wx', f.mode === MODE_EXEC ? EXEC_MODE : FILE_MODE).then(fh => ok(fh), async (e: unknown): Promise<Result<Awaited<ReturnType<typeof open>>, WriteFilesError>> => {
195 await stream.cancel().catch(() => { /* failing: the stream is not read */ });
196 if (thrownCodeIs(e, 'EEXIST')) return kinded(collision(f.path, 'already exists').error, 'collision');
197 const bytes = nameBytes(refused(f.path, e));
198 if (bytes) return bytes;
199 return filesystem(f.path, e);
200 });
201 if (!opened.ok) return stop(opened);
202 const fh = opened.value;
203 try {
204 for await (const c of stream) { let o = 0; while (o < c.length) o += (await fh.write(c, o, c.length - o)).bytesWritten; out.bytes += c.length; }
205 } catch (e) {
206 await fh.close().catch(() => { /* failing: the write error is the one reported */ }); await unlink(full).catch(() => { /* failing: the write error is the one reported */ });
207 return stop(caught(e, f.path));
208 }
209 try { await fh.close(); }
210 catch (e) { return stop(caught(e, f.path)); }
211 out.files++;
212 }
213 return ok(out);
216export type CheckoutDirResult = Omit<CheckoutResult, 'files'> & { readonly written: Written };
217// dir is this function's own refusal. Everything else is writeFiles's Err or checkoutTree's.
218export type CheckoutDirError = WithKind<ErrorOf<'bad_request'>, 'dir'> | WriteFilesError;
219// Absence on a pass-through field is null. ref is a name or null (the remote's HEAD). path '' is the whole tree.
220export type CheckoutDirConfig = { readonly ref: string | null; readonly path: string };
221export type CheckoutDirRequest = { readonly dir: string } & Input<CheckoutDirConfig>;
222export type CheckoutDirOptionsConfig = {
223 readonly auth: HttpAuth | null;
224 readonly parent: Span | null;
225 readonly fetchImpl: NonNullable<ReadOptions['fetchImpl']> | null;
226 readonly limits: LimitsInput;
227 readonly signal: AbortSignal | null;
228 readonly quirks: Quirks | null;
229 readonly transport: Transport | null;
230 readonly batch: number;
231 readonly platform: string;
232};
233// `tracer` is optional, as on every entry: without one operation() drops the spans.
234export type CheckoutDirOptions = Input<{ readonly tracer: Tracer } & CheckoutDirOptionsConfig>;
235type CheckoutDirOptionsTotal = { readonly tracer: Option<Tracer> } & CheckoutDirOptionsConfig;
236const dirDefaults = (): CheckoutDirOptionsConfig => ({
237 auth: null, parent: null, fetchImpl: null, limits: DEFAULT_LIMITS, signal: null, quirks: null, transport: null, batch: CHECKOUT_BATCH, platform: processPlatform,
238});
239// checkoutTree still takes a partial ReadOptions. Null pass-through fields stay off that object.
240function treeOpts(opts: CheckoutDirOptionsTotal): ReadOptions & { readonly batch: number } {
241 const base = {
242 auth: opts.auth,
243 parent: opts.parent,
244 limits: opts.limits,
245 signal: opts.signal,
246 transport: opts.transport,
247 batch: opts.batch,
248 };
249 const traced = opts.tracer.tag === 'some' ? { ...base, tracer: opts.tracer.value } : base;
250 const withFetch = opts.fetchImpl === null ? traced : { ...traced, fetchImpl: opts.fetchImpl };
251 return opts.quirks === null ? withFetch : { ...withFetch, quirks: opts.quirks };
253export async function checkoutDir(url: string, request: CheckoutDirRequest, options: CheckoutDirOptions = {}): Promise<Result<CheckoutDirResult, CheckoutDirError>> {
254 const { dir } = request;
255 const { ref, path } = withDefaults<CheckoutDirConfig>(request, { ref: null, path: '' });
256 // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- a caller's tracer, if any: operation() checks it has start()
257 const opts: CheckoutDirOptionsTotal = { ...withDefaults(options, dirDefaults()), tracer: mapOption(field(options, 'tracer'), t => t as Tracer) };
258 if (typeof dir !== 'string' || dir === '') return kinded(fail('bad_request', 'dir must be a non-empty string', { what: 'dir' }).error, 'dir');
259 // parsed once, for the checkout and the write
260 const limits = parseLimits(opts.limits);
261 if (!limits.ok) return limits;
262 const ready = await emptyDir(dir);
263 if (!ready.ok) return ready;
264 const checked = await checkoutTree(url, { ref, path }, treeOpts({ ...opts, limits: limits.value }));
265 if (!checked.ok) return checked;
266 const { files, ...rest } = checked.value;
267 const written = await writeFiles(dir, files, { limits: limits.value, platform: opts.platform });
268 if (!written.ok) return written;
269 return ok({ ...rest, written: written.value });
Diagnostics6 git calls · 28 ms
CallStatusSizeTime028 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=21
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch2005.7 KiB · hit21 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=5829
git.response.cache=HIT
git.response.duration_ms=21
git.fetch.wants=9b6bdff228ecb68218f42c88d570b6a287084015
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=5829
git.pkt.payload.max=5802
git.sideband.band1.bytes=5802
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f8469bda332c
Request
8e587b52…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format