main
1// Node only (node:fs, node:path): a directory as pushTree's `files`. Not part of any Workers bundle — the Workers-safe
2// core is tree.js, which takes the files as data.
3//
4// filesFromDir(dir, { limits, filter }) → [{ path, mode, data }]
5// pushDir(url, { dir, filter, ...spec }, opts) = pushTree(url, { ...spec, files: await filesFromDir(dir, …) }, opts)
6//
7// Modes as git's index records them (ce_mode_from_stat): a regular file with the owner's execute bit is 100755,
8// otherwise 100644; a symlink is 120000 with its target as the blob (never followed); a directory is a tree (an empty
9// one disappears, as in git). `.git` is skipped at every level. Anything else (FIFO, socket, device) is `bad_request`,
10// not a silent skip. `filter(path, { directory, symlink })` → false leaves a path out (`.gitignore` is not read).
11// Bounds: each file by maxObjectSize (checked on lstat, before the read), the count by maxPackObjects, the bytes held
12// by maxInflatedTotal — the resident set ply already accepts for one buffered pack, LIMIT_OBJECT_OVERHEAD per file included.
13import { readdir, readFile, readlink, lstat } from 'node:fs/promises';
14// oxlint-disable-next-line unicorn/import-style -- `path` names a file's path in the tree throughout this file
15import { join } from 'node:path';
16import { platform } from 'node:process';
17import { callbackFailed, describeThrown, fail } from './errors.js';
18import type { ErrorOf, PlyError } from './errors.js';
19import { DEFAULT_LIMITS, LIMIT_OBJECT_OVERHEAD, limitExceeded, limitsOf, parseLimits } from './limits.js';
20import type { LimitsInput } from './limits.js';
21import { MODE_BLOB, MODE_EXEC, MODE_LINK } from './format.js';
22import { err, ok } from './result.js';
23import type { Result } from './result.js';
24import type { Input } from './absent.js';
25import { isArgObject, withDefaults } from './absent.js';
26import { pushTree } from './tree.js';
27import type { PushTreeOptions, PushTreeResult, TreeSpec } from './tree.js';
29/* oxlint-disable no-await-in-loop -- the walk reads one entry at a time, in sorted order, under the byte bound */
31const OWNER_EXEC = 0o100;
32// The name a tree records for a directory entry. On macOS it is the name `git add` records under the configuration
33// `git init` writes there (core.precomposeunicode = true): the file system keeps a name as it was created, often
34// decomposed, and compat/precompose_utf8.c passes each entry through iconv UTF-8-MAC → UTF-8. That is not Unicode NFC.
35// As git 2.50 does on macOS (test/push-tree.test.js, darwin only, against git add): a code point joins the one recorded
36// just before it when the two are exactly a precomposed character's canonical pair; marks are not reordered; a
37// character standing alone is never rewritten (KELVIN SIGN, U+0958, U+F900 stay); a name holding a code point beyond
38// the BMP is kept as read; the Balinese pairs (U+1B00 block, Unicode 5.0, later than iconv's table) are not joined.
39// Elsewhere it is the name as read.
40const NON_ASCII = /[\u0080-\u{10FFFF}]/u;
41const BEYOND_BMP = /[\u{10000}-\u{10FFFF}]/u;
42function precomposed(name: string): string {
43 if (!NON_ASCII.test(name) || BEYOND_BMP.test(name)) return name;
44 let out = '';
45 for (const ch of name) {
46 const last = out.slice(-1);
47 const pair = (last + ch).normalize('NFC');
48 const joins = last !== '' && pair.length === 1 && last.normalize('NFC') === last
49 && pair.normalize('NFD') === last.normalize('NFD') + ch && (pair < 'ᬀ' || pair > '');
50 out = joins ? out.slice(0, -1) + pair : out + ch;
51 }
52 return out;
53}
54const recorded: (name: string) => string = platform === 'darwin' ? precomposed : name => name;
56// A file of the directory, as pushTree takes it.
57export type DirFile = { readonly path: string; readonly mode: string; readonly data: Uint8Array };
58// `filter(path, { directory, symlink })` → false leaves the path out.
59export type DirFilter = (path: string, kind: { readonly directory: boolean; readonly symlink: boolean }) => boolean;
60// The walk's options, total: the bounds, and the caller's filter (null: keep every path).
61export type FilesFromDirConfig = { readonly limits: LimitsInput; readonly filter: DirFilter | null };
62export type FilesFromDirOptions = Input<FilesFromDirConfig>;
63type DirError = ErrorOf<'bad_request' | 'limit_exceeded'>;
65// The walk as a value: a refused entry or bound is an Err, and so is the file system's own failure (a missing
66// directory, a permission): bad_request naming 'dir', with node:fs's error (its errno `code`, `path`) as `cause`.
67// Definite: the walk precedes any request. filesFromDir never rejects.
68export async function filesFromDir(dir: string, options: FilesFromDirOptions | null = null): Promise<Result<DirFile[], DirError>> {
69 if (options !== null && !isArgObject(options)) return fail('bad_request', 'filesFromDir: options must be an object or null', { what: 'options' });
70 const { limits, filter } = withDefaults<FilesFromDirConfig>(options, { limits: DEFAULT_LIMITS, filter: null });
71 const checked = parseLimits(limits);
72 if (!checked.ok) return checked;
73 const { maxObjectSize: maxObject, maxPackObjects: maxFiles, maxInflatedTotal: maxTotal } = checked.value;
74 const out: DirFile[] = [];
75 let total = 0;
76 const keep = (data: Uint8Array, path: string, mode: string): Result<null, DirError> => {
77 if (out.length >= maxFiles) return err(limitExceeded('files', out.length + 1, maxFiles));
78 // each file costs LIMIT_OBJECT_OVERHEAD beyond its bytes (#142)
79 total += data.length + LIMIT_OBJECT_OVERHEAD;
80 if (total > maxTotal) return err(limitExceeded('directory bytes', total, maxTotal));
81 out.push({ path, mode, data: new Uint8Array(data.buffer, data.byteOffset, data.length) });
82 return ok(null);
83 };
84 // One entry: the directory to walk into, the file or link to keep, or the refusal.
85 // `disk` is the directory as the file system names it (what is opened), `rel` the same directory as recorded.
86 const walk = async (disk: string, rel: string): Promise<Result<null, DirError>> => {
87 const names = (await readdir(disk ? join(dir, disk) : dir)).toSorted();
88 for (const name of names) {
89 if (name === '.git') continue;
90 const path = rel ? `${rel}/${recorded(name)}` : recorded(name);
91 const onDisk = disk ? `${disk}/${name}` : name;
92 const full = join(dir, onDisk);
93 const st = await lstat(full);
94 // the caller's filter: what it throws is an Err naming 'filter' (definite: the walk precedes any request)
95 let wanted = true;
96 if (filter !== null) { try { wanted = filter(path, { directory: st.isDirectory(), symlink: st.isSymbolicLink() }); } catch (e) { return err(callbackFailed('filter', e)); } }
97 if (!wanted) continue;
98 let kept: Result<null, DirError>;
99 if (st.isDirectory()) kept = await walk(onDisk, path);
100 else if (st.isSymbolicLink()) kept = keep(await readlink(full, { encoding: 'buffer' }), path, MODE_LINK);
101 else if (st.isFile()) {
102 if (st.size > maxObject) return err(limitExceeded('object size', st.size, maxObject));
103 kept = keep(await readFile(full), path, (st.mode & OWNER_EXEC) === 0 ? MODE_BLOB : MODE_EXEC);
104 } else return fail('bad_request', `${path}: not a regular file, directory or symlink`, { what: 'path', path });
105 if (!kept.ok) return kept;
106 }
107 return ok(null);
108 };
109 let walked: Result<null, DirError>;
110 try { walked = await walk('', ''); } catch (e) {
111 // `dir` is the caller's value: one that is not a string (node:fs refused it) may have no string form to print
112 return fail('bad_request', `${typeof dir === 'string' ? dir : 'dir'}: the directory could not be read: ${describeThrown(e)}`, { what: 'dir', cause: e });
113 }
114 return walked.ok ? ok(out) : walked;
115}
117export type PushDirSpec = Omit<TreeSpec, 'files'> & { readonly dir: string } & Input<{ readonly filter: DirFilter | null }>;
118export async function pushDir(url: string, given: PushDirSpec, opts: PushTreeOptions = {}): Promise<Result<PushTreeResult, PlyError>> {
119 if (!isArgObject(given)) return fail('bad_request', 'pushDir: spec must be an object', { what: 'spec' });
120 if (!isArgObject(opts)) return fail('bad_request', 'pushDir: options must be an object', { what: 'options' });
121 const { dir, filter = null, ...spec } = given;
122 const limits = limitsOf(opts);
123 if (!limits.ok) return limits;
124 const files = await filesFromDir(dir, { limits: limits.value, filter });
125 if (!files.ok) return files;
126 return pushTree(url, { ...spec, files: files.value }, { ...opts, limits: limits.value });
127}
Diagnostics4 git calls · 414 ms
CallStatusSizeTime0414 ms
discover200152 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200145 B · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=145 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=145 git.pkt.payload.max=76 git.ls_refs.count=2 git.ls_refs.head=refs/heads/main
fetchblob:none · deepen 1020038 KiB · hit7 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=404 git.response.bytes=38434 git.response.cache=HIT git.response.duration_ms=7 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=6 git.pkt.data=4 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=38434 git.pkt.payload.max=38339 git.sideband.band1.bytes=38338 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=63 git.pack.bytes=38338 git.pack.ofs_delta=0 git.pack.ref_delta=8 git.pack.chain.max=8 git.pack.trailer=0bc36ab7adf82d3109177f11f1ea93f1ce071f77 git.pack.full=55Open pack map
fetch2003.4 KiB404 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3439 git.response.cache=MISS git.response.duration_ms=404 git.fetch.wants=8d4ebca46d36c776c57e99d45e735c5634eeb7a2 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3439 git.pkt.payload.max=3418 git.sideband.band1.bytes=3417 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/ply.git
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e1afa48a5cf54b
- Request
- 96ba2ce3…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format