GitHubivarvong

ply

main
ply6 git calls186 ms23 KiB4 cachedprotocol v2
127 lines8.3 KiBRaw
1// Node only (node:fs, node:path): a directory as pushTree's `files`. Not part of any Workers bundle — the Workers-safe
2// core is tree.js, which takes the files as data.
3//
4// filesFromDir(dir, { limits, filter }) → [{ path, mode, data }]
5// pushDir(url, { dir, filter, ...spec }, opts) = pushTree(url, { ...spec, files: await filesFromDir(dir, …) }, opts)
6//
7// Modes as git's index records them (ce_mode_from_stat): a regular file with the owner's execute bit is 100755,
8// otherwise 100644; a symlink is 120000 with its target as the blob (never followed); a directory is a tree (an empty
9// one disappears, as in git). `.git` is skipped at every level. Anything else (FIFO, socket, device) is `bad_request`,
10// not a silent skip. `filter(path, { directory, symlink })` → false leaves a path out (`.gitignore` is not read).
11// Bounds: each file by maxObjectSize (checked on lstat, before the read), the count by maxPackObjects, the bytes held
12// by maxInflatedTotal — the resident set ply already accepts for one buffered pack, LIMIT_OBJECT_OVERHEAD per file included.
13import { readdir, readFile, readlink, lstat } from 'node:fs/promises';
14// oxlint-disable-next-line unicorn/import-style -- `path` names a file's path in the tree throughout this file
15import { join } from 'node:path';
16import { platform } from 'node:process';
17import { callbackFailed, describeThrown, fail } from './errors.js';
18import type { ErrorOf, PlyError } from './errors.js';
19import { DEFAULT_LIMITS, LIMIT_OBJECT_OVERHEAD, limitExceeded, limitsOf, parseLimits } from './limits.js';
20import type { LimitsInput } from './limits.js';
21import { MODE_BLOB, MODE_EXEC, MODE_LINK } from './format.js';
22import { err, ok } from './result.js';
23import type { Result } from './result.js';
24import type { Input } from './absent.js';
25import { isArgObject, withDefaults } from './absent.js';
26import { pushTree } from './tree.js';
27import type { PushTreeOptions, PushTreeResult, TreeSpec } from './tree.js';
29/* oxlint-disable no-await-in-loop -- the walk reads one entry at a time, in sorted order, under the byte bound */
31const OWNER_EXEC = 0o100;
32// The name a tree records for a directory entry. On macOS it is the name `git add` records under the configuration
33// `git init` writes there (core.precomposeunicode = true): the file system keeps a name as it was created, often
34// decomposed, and compat/precompose_utf8.c passes each entry through iconv UTF-8-MAC → UTF-8. That is not Unicode NFC.
35// As git 2.50 does on macOS (test/push-tree.test.js, darwin only, against git add): a code point joins the one recorded
36// just before it when the two are exactly a precomposed character's canonical pair; marks are not reordered; a
37// character standing alone is never rewritten (KELVIN SIGN, U+0958, U+F900 stay); a name holding a code point beyond
38// the BMP is kept as read; the Balinese pairs (U+1B00 block, Unicode 5.0, later than iconv's table) are not joined.
39// Elsewhere it is the name as read.
40const NON_ASCII = /[\u0080-\u{10FFFF}]/u;
41const BEYOND_BMP = /[\u{10000}-\u{10FFFF}]/u;
42function precomposed(name: string): string {
43 if (!NON_ASCII.test(name) || BEYOND_BMP.test(name)) return name;
44 let out = '';
45 for (const ch of name) {
46 const last = out.slice(-1);
47 const pair = (last + ch).normalize('NFC');
48 const joins = last !== '' && pair.length === 1 && last.normalize('NFC') === last
49 && pair.normalize('NFD') === last.normalize('NFD') + ch && (pair < 'ᬀ' || pair > '᭿');
50 out = joins ? out.slice(0, -1) + pair : out + ch;
51 }
52 return out;
54const recorded: (name: string) => string = platform === 'darwin' ? precomposed : name => name;
56// A file of the directory, as pushTree takes it.
57export type DirFile = { readonly path: string; readonly mode: string; readonly data: Uint8Array };
58// `filter(path, { directory, symlink })` → false leaves the path out.
59export type DirFilter = (path: string, kind: { readonly directory: boolean; readonly symlink: boolean }) => boolean;
60// The walk's options, total: the bounds, and the caller's filter (null: keep every path).
61export type FilesFromDirConfig = { readonly limits: LimitsInput; readonly filter: DirFilter | null };
62export type FilesFromDirOptions = Input<FilesFromDirConfig>;
63type DirError = ErrorOf<'bad_request' | 'limit_exceeded'>;
65// The walk as a value: a refused entry or bound is an Err, and so is the file system's own failure (a missing
66// directory, a permission): bad_request naming 'dir', with node:fs's error (its errno `code`, `path`) as `cause`.
67// Definite: the walk precedes any request. filesFromDir never rejects.
68export async function filesFromDir(dir: string, options: FilesFromDirOptions | null = null): Promise<Result<DirFile[], DirError>> {
69 if (options !== null && !isArgObject(options)) return fail('bad_request', 'filesFromDir: options must be an object or null', { what: 'options' });
70 const { limits, filter } = withDefaults<FilesFromDirConfig>(options, { limits: DEFAULT_LIMITS, filter: null });
71 const checked = parseLimits(limits);
72 if (!checked.ok) return checked;
73 const { maxObjectSize: maxObject, maxPackObjects: maxFiles, maxInflatedTotal: maxTotal } = checked.value;
74 const out: DirFile[] = [];
75 let total = 0;
76 const keep = (data: Uint8Array, path: string, mode: string): Result<null, DirError> => {
77 if (out.length >= maxFiles) return err(limitExceeded('files', out.length + 1, maxFiles));
78 // each file costs LIMIT_OBJECT_OVERHEAD beyond its bytes (#142)
79 total += data.length + LIMIT_OBJECT_OVERHEAD;
80 if (total > maxTotal) return err(limitExceeded('directory bytes', total, maxTotal));
81 out.push({ path, mode, data: new Uint8Array(data.buffer, data.byteOffset, data.length) });
82 return ok(null);
83 };
84 // One entry: the directory to walk into, the file or link to keep, or the refusal.
85 // `disk` is the directory as the file system names it (what is opened), `rel` the same directory as recorded.
86 const walk = async (disk: string, rel: string): Promise<Result<null, DirError>> => {
87 const names = (await readdir(disk ? join(dir, disk) : dir)).toSorted();
88 for (const name of names) {
89 if (name === '.git') continue;
90 const path = rel ? `${rel}/${recorded(name)}` : recorded(name);
91 const onDisk = disk ? `${disk}/${name}` : name;
92 const full = join(dir, onDisk);
93 const st = await lstat(full);
94 // the caller's filter: what it throws is an Err naming 'filter' (definite: the walk precedes any request)
95 let wanted = true;
96 if (filter !== null) { try { wanted = filter(path, { directory: st.isDirectory(), symlink: st.isSymbolicLink() }); } catch (e) { return err(callbackFailed('filter', e)); } }
97 if (!wanted) continue;
98 let kept: Result<null, DirError>;
99 if (st.isDirectory()) kept = await walk(onDisk, path);
100 else if (st.isSymbolicLink()) kept = keep(await readlink(full, { encoding: 'buffer' }), path, MODE_LINK);
101 else if (st.isFile()) {
102 if (st.size > maxObject) return err(limitExceeded('object size', st.size, maxObject));
103 kept = keep(await readFile(full), path, (st.mode & OWNER_EXEC) === 0 ? MODE_BLOB : MODE_EXEC);
104 } else return fail('bad_request', `${path}: not a regular file, directory or symlink`, { what: 'path', path });
105 if (!kept.ok) return kept;
106 }
107 return ok(null);
108 };
109 let walked: Result<null, DirError>;
110 try { walked = await walk('', ''); } catch (e) {
111 // `dir` is the caller's value: one that is not a string (node:fs refused it) may have no string form to print
112 return fail('bad_request', `${typeof dir === 'string' ? dir : 'dir'}: the directory could not be read: ${describeThrown(e)}`, { what: 'dir', cause: e });
113 }
114 return walked.ok ? ok(out) : walked;
117export type PushDirSpec = Omit<TreeSpec, 'files'> & { readonly dir: string } & Input<{ readonly filter: DirFilter | null }>;
118export async function pushDir(url: string, given: PushDirSpec, opts: PushTreeOptions = {}): Promise<Result<PushTreeResult, PlyError>> {
119 if (!isArgObject(given)) return fail('bad_request', 'pushDir: spec must be an object', { what: 'spec' });
120 if (!isArgObject(opts)) return fail('bad_request', 'pushDir: options must be an object', { what: 'options' });
121 const { dir, filter = null, ...spec } = given;
122 const limits = limitsOf(opts);
123 if (!limits.ok) return limits;
124 const files = await filesFromDir(dir, { limits: limits.value, filter });
125 if (!files.ok) return files;
126 return pushTree(url, { ...spec, files: files.value }, { ...opts, limits: limits.value });
Diagnostics6 git calls · 186 ms
CallStatusSizeTime0186 ms
discover200191 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=191
git.server.agent=git/github-9922ec520325-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=191
git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=11990
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=152
git.pkt.data=151
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=11990
git.pkt.payload.max=122
git.ls_refs.count=151
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4347
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4347
git.pkt.payload.max=4247
git.sideband.band1.bytes=4247
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=10
git.pack.bytes=4247
git.pack.ofs_delta=0
git.pack.ref_delta=0
git.pack.chain.max=0
git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400
git.pack.full=10
Open pack map
fetchtree:0200919 B · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=919
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=919
git.pkt.payload.max=892
git.sideband.band1.bytes=892
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=892
git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76e
Open pack map
fetchtree:02002.4 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=177
git.response.bytes=2507
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2507
git.pkt.payload.max=2480
git.sideband.band1.bytes=2480
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=2480
git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381
Open pack map
fetch2003.4 KiB177 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3448
git.response.cache=MISS
git.response.duration_ms=177
git.fetch.wants=8d4ebca46d36c776c57e99d45e735c5634eeb7a2
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3448
git.pkt.payload.max=3421
git.sideband.band1.bytes=3421
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://github.com/ivarvong/ply
Ref
refs/heads/main
Commit
19912563f540228593df0c489082e5e2508e21e4
cf-ray
a5e0f8272cfc332c
Request
aee591c3…
Protocol
v2 · sha1
Agent
git/github-9922ec520325-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format