GitHubivarvong
ply
main
1// Multi-object fetch: the pack read through the scan as its band-1 bytes arrive (bounded by limits.maxPackBytes, enforced
2// by v2Step), its objects collected and delta-resolved. The pack's bytes and its zlib members are dropped once inflated,
3// so what is held is the objects (#172).
4// The general-purpose path (clones, thin packs, negotiation); the edit pipeline uses fetch.js's fetchObject instead.
5// Reading the received pack (readPack), fetchPack and its fallbacks return Result.
6import type { Span } from './otel.js';
7import type { Attrs, Transport } from './transport.js';
8import type { FetchOpts, PackObject, V2State, WideFilter } from './fetch.js';
9import type { Input } from './absent.js';
10import { TYPE, TYPE_NAME } from './objects.js';
11import { resolvePack, packCaps, packCollectOpen } from './packresolve.js';
12import type { ObjBase, ParsedPack, ResolvedPack } from './packresolve.js';
13import { packStream, collectWire, isFilterRejection, filterUnadvertised, fallbackName, harvest, checkWide, skippedAttrs, fetchExchange } from './fetch.js';
14import { fail } from './errors.js';
15import { probing } from './transport.js';
16import type { PlyError } from './errors.js';
17import { fromNullable, withDefaults } from './absent.js';
18import { ok, assertNever } from './result.js';
19import type { Result } from './result.js';
20// oxlint-disable-next-line import/max-dependencies -- the buffered fetch joins the stream, the pack parser and the cache's retry rule
21import { corruptCacheHit } from './cachestatus.js';
23// The pack and its resolution, as packresolve.ts declares them.
24type Parsed = ParsedPack;
25type Resolved = ResolvedPack;
26// A thin pack's base by oid, or null when the caller does not hold it.
27export type ExternalLookup = (oid: string) => PackObject | null;
28// `external(oid)` supplies a thin pack's bases; `check(objects)` runs over the resolved objects inside the span: its
29// attrs go on the span, its Err is the fetch's.
30export type FetchPackConfig = {
31 readonly external: ExternalLookup;
32 readonly resolve: boolean;
33 readonly attrs: Attrs;
34 readonly check: ((objects: Map<string, PackObject>) => Result<Attrs, PlyError>) | null;
35};
36export type FetchPackExtra = Input<FetchPackConfig>;
37const noExternal: ExternalLookup = () => null;
38const fetchPackDefaults = (): FetchPackConfig => ({ external: noExternal, resolve: true, attrs: {}, check: null });
39export type FetchPackResult = {
40 readonly parsed: Parsed | null; readonly objects: Map<string, PackObject> | null; readonly resolved: Resolved | null;
41 readonly acks: string[]; readonly nak: boolean; readonly ready: boolean; readonly shallow: readonly string[]; readonly unshallow: readonly string[];
42};
44export const packAttrs = (parsed: Parsed, resolved: Resolved | null): Attrs => ({
45 'git.pack.version': parsed.stats.version, 'git.pack.objects': parsed.stats.count, 'git.pack.bytes': parsed.bytes,
46 'git.pack.trailer': parsed.trailer, 'git.pack.full': parsed.stats.full, 'git.pack.ofs_delta': parsed.stats.ofsDelta,
47 'git.pack.ref_delta': parsed.stats.refDelta, 'git.pack.compressed.bytes': parsed.stats.compressed, 'git.pack.inflated.bytes': parsed.stats.inflated,
48 'git.pack.inflate.blocks': parsed.stats.inflate.blocks, 'git.pack.inflate.stored': parsed.stats.inflate.stored,
49 'git.pack.inflate.fixed': parsed.stats.inflate.fixed, 'git.pack.inflate.dynamic': parsed.stats.inflate.dynamic,
50 ...Object.fromEntries(Object.entries(parsed.stats.byType).map(([k, v]) => [`git.pack.type.${k}`, v])),
51 ...(resolved ? { 'git.pack.chain.max': resolved.stats.maxChain, 'git.pack.external_bases': resolved.stats.externalBases } : {}),
52});
54// The span attribute for a type id: its name, or nothing for an id objects.js does not know (the attribute is dropped).
55const typeAttr = (type: number): unknown => Reflect.get(TYPE_NAME, type);
56type Acks = Pick<FetchPackResult, 'acks' | 'nak' | 'ready' | 'shallow' | 'unshallow'>;
57// An object's base as its span event names it: the OFS offset (-1 for none) and the REF id ('' for none).
58function baseAttrs(base: ObjBase): Attrs {
59 switch (base.tag) {
60 case 'none': return { 'git.object.base.offset': -1, 'git.object.base.oid': '' };
61 case 'ofs': return { 'git.object.base.offset': base.offset, 'git.object.base.oid': '' };
62 case 'ref': return { 'git.object.base.offset': -1, 'git.object.base.oid': base.oid };
63 }
64 // Stryker disable next-line ConditionalExpression,BlockStatement,CallExpression: unreachable, the compiler's exhaustiveness check
65 return assertNever(base);
66}
67// The received pack, parsed (one span event per object), resolved when asked, and checked.
68function readPack(t: Transport, parsed: Parsed, { external, resolve, check }: FetchPackConfig, acks: Acks, span: Span): Result<FetchPackResult, PlyError> {
69 for (const o of parsed.objects) span.event('git.pack.object', {
70 'git.object.offset': o.offset, 'git.object.type': typeAttr(o.type), 'git.object.size': o.size, 'git.object.compressed': o.compressed,
71 'git.object.header_bytes': o.headerBytes, ...baseAttrs(o.base),
72 'git.object.inflate.blocks': o.inflate.blocks, 'git.object.inflate.stored': o.inflate.stored, 'git.object.inflate.fixed': o.inflate.fixed,
73 'git.object.inflate.dynamic': o.inflate.dynamic, 'git.object.inflate.matches': o.inflate.matches, 'git.object.inflate.max_dist': o.inflate.maxDist,
74 });
75 let resolved: Resolved | null = null;
76 if (resolve) {
77 const r = resolvePack(parsed, external, t.limits);
78 if (!r.ok) return r;
79 resolved = r.value;
80 }
81 span.set(packAttrs(parsed, resolved));
82 if (check && resolved) {
83 const checked = check(resolved.objects);
84 if (!checked.ok) return checked;
85 span.set(checked.value);
86 }
87 return ok({ parsed, objects: resolved === null ? null : resolved.objects, resolved, ...acks });
88}
90// `check(objects)`, when given, runs over the resolved objects inside the span (it may drop some, or fail) and returns
91// attrs for it.
92// oxlint-disable-next-line require-await -- async: a synchronous failure (an unusable transport) is a rejection, as it always was
93export async function fetchPackOnce(t: Transport, opts: FetchOpts, extra: FetchPackExtra = {}): Promise<Result<FetchPackResult, PlyError>> {
94 const cfg = withDefaults(extra, fetchPackDefaults());
95 return fetchExchange(t, { 'git.pack.cache.bust': Boolean(opts.bypassCache), ...cfg.attrs }, async (span: Span): Promise<Result<FetchPackResult, PlyError>> => {
96 const caps = packCaps({ limits: t.limits });
97 if (!caps.ok) return caps;
98 const c = packCollectOpen(caps.value, true);
99 // the caps passed packCaps, so the scan opens (packScanOpen)
100 // Stryker disable next-line ConditionalExpression,BlockStatement: see above
101 if (!c.ok) return c;
102 const read = await collectWire(packStream(t, opts, span), c.value);
103 const st: V2State = read.state;
104 const acks: Acks = { acks: st.acks.filter(a => a.startsWith('ACK ')).map(a => a.slice(4)), nak: st.acks.includes('NAK'), ready: st.ready, shallow: st.shallow, unshallow: st.unshallow };
105 if (st.phase !== 'pack') { span.set({ 'git.fetch.pack': false }); return ok({ parsed: null, objects: null, resolved: null, ...acks }); }
106 if (!read.pack.ok) return read.pack;
107 return readPack(t, read.pack.value, cfg, acks, span);
108 });
109}
111// A pack that fails verification may be a poisoned cache entry (see packcache.js): retry once, bypassing the cache.
112// Only when the bad response was served from cache; a corrupt live response is reported as-is.
113export async function fetchPack(t: Transport, opts: FetchOpts, extra: FetchPackExtra = {}): Promise<Result<FetchPackResult, PlyError>> {
114 const first = await fetchPackOnce(t, opts, extra);
115 if (first.ok || !corruptCacheHit(first.error) || Boolean(opts.bypassCache)) return first;
116 return fetchPackOnce(t, { ...opts, bypassCache: true }, extra);
117}
119// fetchPack for a `filter tree:0` request (commit-only history walks) with the blob:none fallback of fetch.js: a server
120// that refuses tree:0 gets blob:none, which also carries every tree of every commit in the window — far more bytes on a
121// deep window (bounded by limits.maxPackBytes; the attrs say how much). The trees are harvested for later tree fetches.
122// Every commit, tree and tag in the pack is checked as fetchObject checks one (assertFetched), inside the fetch's span,
123// so a malformed object fails the fetch (`pack_corrupt`) instead of reaching a caller or the harvest. On the tree:0
124// path all of it is what was asked for. On the blob:none path the trees are extras a tree:0 server would not have sent
125// (the walk reads commits only): checkWide drops one that is over a limit instead of failing the walk (#46; see there).
126// A server whose advertisement has no `filter` (git's default, fetch.js filterUnadvertised) gets no filter: the pack then
127// carries every tree and blob of the window too, held to the same limits; the trees are checked and harvested the same
128// way, the blobs are hash-checked by the resolver and dropped (the walk reads commits only).
129// The result carries `filter` (the one the answered request used; null for none) and `learned` (true only on the
130// request that discovered the refusal; a later call on the same quirks asks for the fallback directly and reports
131// learned false).
132const commitsAndTags = (_oid: string, o: PackObject): boolean => o.type !== TYPE.TREE;
133export type Tree0Pack = FetchPackResult & { readonly filter: string | null; readonly learned: boolean };
134// oxlint-disable-next-line typescript/promise-function-async -- probing's promise, passed through
135export function fetchPackTree0(t: Transport, opts: FetchOpts): Promise<Result<Tree0Pack, PlyError>> {
136 const wide = async (filter: WideFilter, learned: boolean): Promise<Result<Tree0Pack, PlyError>> => {
137 const check = (objects: Map<string, PackObject>): Result<Attrs, PlyError> => { const c = checkWide(objects, commitsAndTags, t.limits); return c.ok ? ok(skippedAttrs(c.value)) : c; };
138 const r = await fetchPack(t, { ...opts, filter }, { check, attrs: { 'git.fetch.filter.fallback': fallbackName(filter), 'git.fetch.filter.requested': opts.filter, 'git.fetch.filter.learned': learned } });
139 if (!r.ok) return r;
140 if (r.value.objects) {
141 const h = harvest(t, r.value.objects);
142 if (!h.ok) return h;
143 }
144 return ok({ ...r.value, filter: filter === '' ? null : filter, learned });
145 };
146 // one probe at a time per quirks record (transport.ts probing): the checks below read it after any wait
147 return probing(t.quirks, opts.filter === 'tree:0' ? 'tree:0' : 'filter', async (): Promise<Result<Tree0Pack, PlyError>> => {
148 if (t.quirks.treeFilter === 'none') return wide('', false);
149 if (t.quirks.treeFilter === 'blob:none') return wide('blob:none', false);
150 const base = await fetchPack(t, opts, { check: (objects): Result<Attrs, PlyError> => { const c = checkWide(objects, () => true, t.limits); return c.ok ? ok({}) : c; } });
151 if (base.ok) {
152 const f = fromNullable(opts.filter);
153 const filterVal = f.tag === 'some' ? f.value : null;
154 return ok({ ...base.value, filter: filterVal, learned: false });
155 }
156 if (isFilterRejection(base.error)) {
157 t.quirks.treeFilter = 'blob:none';
158 return wide('blob:none', true);
159 }
160 return (await filterUnadvertised(t, base.error)) ? wide('', true) : base;
161 });
162}
164// The two filters a history walk asks for. Anything else is the caller's bug, refused before a request.
165export type PackFilter = 'tree:0' | 'blob:none';
166const isPackFilter = (f: unknown): f is PackFilter => f === 'tree:0' || f === 'blob:none';
167export type FilteredWantConfig = { readonly wants: readonly string[]; readonly filter: PackFilter; readonly deepen: number };
168export type FilteredWant = Input<FilteredWantConfig>;
169// `filter` is the one the answered request used: the asked one, blob:none (a host that refuses tree:0), or null (a host
170// that does not take a filter at all: the pack is unfiltered).
171export type FilteredPack = {
172 readonly objects: Map<string, PackObject> | null;
173 readonly shallow: readonly string[];
174 readonly filter: PackFilter | null;
175 readonly learned: boolean;
176};
177// A caller's value, quoted: a primitive as JSON, anything else (the platform's missing value included) by its typeof.
178const shown = (v: unknown): string => (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null ? JSON.stringify(v) : typeof v);
179// fetchPackTree0 as a Result, for a caller that already holds the transport (ply-web's loadPack / loadWindow).
180// `learned` is true only when this call is the one that found the server refuses tree:0.
181export async function fetchPackFiltered(t: Transport, opts: FilteredWant | null): Promise<Result<FilteredPack, PlyError>> {
182 if (typeof opts !== 'object' || opts === null) return fail('bad_request', 'fetchPackFiltered: expected { wants, filter, deepen }', { what: 'opts' });
183 const { filter, wants } = opts;
184 if (!isPackFilter(filter)) return fail('bad_request', `fetchPackFiltered: filter must be tree:0 or blob:none, got ${shown(filter)}`, { what: 'filter' });
185 if (!Array.isArray(wants)) return fail('bad_request', 'fetchPackFiltered: wants must be a list of object ids', { what: 'wants' });
186 if (wants.length === 0) return fail('bad_request', 'fetchPackFiltered: wants must name at least one object', { what: 'wants' });
187 const r = await fetchPackTree0(t, { wants, filter, deepen: opts.deepen });
188 if (!r.ok) return r;
189 // Success echoes the filter we just checked, the fallback's blob:none, or null for none.
190 const answered = fromNullable(r.value.filter);
191 let echoed: PackFilter | null = null;
192 if (answered.tag === 'some') echoed = answered.value === 'blob:none' ? 'blob:none' : filter;
193 return ok({ objects: r.value.objects, shallow: r.value.shallow, filter: echoed, learned: r.value.learned });
194}
Diagnostics6 git calls · 30 ms
CallStatusSizeTime030 ms
discover200191 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=191 git.server.agent=git/github-9922ec520325-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=191 git.pkt.payload.max=36
ls-refs20012 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=11990 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=152 git.pkt.data=151 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=11990 git.pkt.payload.max=122 git.ls_refs.count=151 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 102004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4347 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=19912563f540228593df0c489082e5e2508e21e4 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4347 git.pkt.payload.max=4247 git.sideband.band1.bytes=4247 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=10 git.pack.bytes=4247 git.pack.ofs_delta=0 git.pack.ref_delta=0 git.pack.chain.max=0 git.pack.trailer=a6caf8e129bea82c9bae4cbc7c130c9205add400 git.pack.full=10Open pack map
fetchtree:0200919 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=919 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=800d30ba8923860bfc460ca884b7f9ba22083588 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=919 git.pkt.payload.max=892 git.sideband.band1.bytes=892 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=892 git.pack.trailer=5323b9266eea5ded637285d1c5320699f12be76eOpen pack map
fetchtree:02002.4 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=22 git.response.bytes=2507 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=8f720d29ff85ce682ffdad9c393b04a3834c3203 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2507 git.pkt.payload.max=2480 git.sideband.band1.bytes=2480 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=2480 git.pack.trailer=723b6bf1d4d6021fe5eb940c622b9ff00ee4a381Open pack map
fetch2004.6 KiB · hit22 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=4706 git.response.cache=HIT git.response.duration_ms=22 git.fetch.wants=26b37d78148b7ceea8082e147dd7bda04ed69b83 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4706 git.pkt.payload.max=4679 git.sideband.band1.bytes=4679 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://github.com/ivarvong/ply
- Ref
- refs/heads/main
- Commit
- 19912563f540228593df0c489082e5e2508e21e4
- cf-ray
- a5e0f8405ea2332c
- Request
- 59336109…
- Protocol
- v2 · sha1
- Agent
- git/github-9922ec520325-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format