GitLabgitlab-org

cli

main
ply7 git calls29 ms99 KiB5 cachedprotocol v2
163 lines6.0 KiBRaw
1package proxy
3import (
4 "io"
5 "os"
7 "gitlab.com/gitlab-org/cli/internal/dbg"
8)
10// peekLimits bounds how much of an upload body is inspected.
11//
12// inMemory is the amount of an upload body buffered in memory to extract the
13// package coordinate. npm packuments and twine multipart field parts live near
14// the front of the body, so this is normally ample; larger bodies spill to a
15// temp file rather than failing to match.
16//
17// max is the hard ceiling on an inspected upload body. A body larger than this
18// is treated as un-inspectable and the request fails closed (blocked) instead
19// of being passed through un-checked.
20type peekLimits struct {
21 inMemory int64
22 max int64
25// defaultPeekLimits is the production configuration. Tests pass smaller values
26// to exercise the spill and over-limit paths without allocating gigabytes.
27var defaultPeekLimits = peekLimits{
28 inMemory: 64 << 20, // 64 MiB
29 max: 1 << 30, // 1 GiB
32// orDefault returns l when it is fully specified, otherwise defaultPeekLimits.
33// A zero-value peekLimits (from a matcher constructed as NPMMatcher{}) resolves
34// to the production limits.
35func (l peekLimits) orDefault() peekLimits {
36 if l.inMemory <= 0 || l.max <= 0 {
37 return defaultPeekLimits
38 }
39 return l
42// peekResult carries the outcome of buffering an upload body for inspection.
43type peekResult struct {
44 // data is the bytes available to the matcher for coordinate extraction.
45 data []byte
46 // body is the restored request body the caller must assign back to
47 // req.Body so the upstream round trip still sees the full payload.
48 body io.ReadCloser
49 // overLimit is true when the body exceeded limits.max and could not
50 // be fully inspected; the caller must fail closed (block) in that case.
51 overLimit bool
52 // spilled reads the whole body back from the spill file when it was
53 // larger than limits.inMemory, for matchers whose coordinate can sit past
54 // data. It must be read before Match returns: the upstream round trip
55 // consumes and removes the same file. nil when the body fit in memory.
56 spilled *io.SectionReader
59// peekUploadBody buffers an upload body for coordinate inspection. Bodies up
60// to limits.inMemory are held in memory; larger bodies spill to a temp
61// file (removed when the returned body is closed) so the matcher can read
62// the whole payload without exhausting memory. A body larger than
63// limits.max sets overLimit so the caller fails closed. The returned
64// body always restores the full original stream for the upstream round trip.
65func peekUploadBody(original io.ReadCloser, limits peekLimits) peekResult {
66 mem, err := io.ReadAll(io.LimitReader(original, limits.inMemory))
67 if err != nil {
68 return peekResult{data: mem, body: bodyWithPrefix(mem, original)}
69 }
71 // Probe one more byte: if the body ends at or below the in-memory limit
72 // we can inspect and restore it entirely from memory.
73 var probe [1]byte
74 n, _ := io.ReadFull(original, probe[:])
75 if n == 0 {
76 return peekResult{data: mem, body: bodyWithPrefix(mem, original)}
77 }
79 // The body is larger than the in-memory window; spill the whole payload
80 // to a temp file so we can both inspect it and stream it upstream.
81 f, err := os.CreateTemp("", "glab-df-upload-*")
82 if err != nil {
83 dbg.Debugf("dependency firewall: failed to create upload spill file: %v", err)
84 // Cannot inspect; fail closed.
85 return peekResult{overLimit: true, body: bodyWithPrefix(append(mem, probe[:n]...), original)}
86 }
88 if _, err := f.Write(mem); err != nil {
89 return spillError(f, err, mem, probe[:n], original)
90 }
91 if _, err := f.Write(probe[:n]); err != nil {
92 return spillError(f, err, mem, probe[:n], original)
93 }
95 written := int64(len(mem) + n)
96 remaining := limits.max - written
97 copied, err := io.Copy(f, io.LimitReader(original, remaining+1))
98 if err != nil {
99 // The mem+probe bytes are already in the temp file (now abandoned) and
100 // cannot be recovered into the restored body, so the body spillError
101 // builds here is missing that prefix and is NOT safe to forward. This
102 // is only safe because spillError sets overLimit, and an over-limit
103 // upload is always blocked before the body would be sent upstream. If a
104 // future caller ever forwards this body without honoring overLimit, it
105 // would send a corrupt (truncated-at-the-front) payload.
106 return spillError(f, err, nil, nil, original)
107 }
108 written += copied
109 _ = original.Close()
111 overLimit := written > limits.max
112 if overLimit {
113 dbg.Debugf("dependency firewall: upload body exceeds %d bytes; failing closed", limits.max)
114 }
116 if _, err := f.Seek(0, io.SeekStart); err != nil {
117 _ = f.Close()
118 _ = os.Remove(f.Name())
119 return peekResult{overLimit: true}
120 }
122 data, err := io.ReadAll(io.LimitReader(f, limits.inMemory))
123 if err != nil {
124 _ = f.Close()
125 _ = os.Remove(f.Name())
126 return peekResult{overLimit: true}
127 }
128 if _, err := f.Seek(0, io.SeekStart); err != nil {
129 _ = f.Close()
130 _ = os.Remove(f.Name())
131 return peekResult{overLimit: true}
132 }
134 return peekResult{data: data, body: tempFileBody(f), overLimit: overLimit, spilled: io.NewSectionReader(f, 0, written)}
137// spillError abandons a temp-file spill on write failure, restoring whatever
138// was read so the caller can still forward the body, and signals fail-closed.
139func spillError(f *os.File, err error, mem, probe []byte, original io.ReadCloser) peekResult {
140 dbg.Debugf("dependency firewall: failed to spill upload body: %v", err)
141 _ = f.Close()
142 _ = os.Remove(f.Name())
143 prefix := append(append([]byte{}, mem...), probe...)
144 return peekResult{overLimit: true, body: bodyWithPrefix(prefix, original)}
147// tempFileBody returns a ReadCloser that streams f from its current offset
148// and removes the backing temp file when closed.
149func tempFileBody(f *os.File) io.ReadCloser {
150 return &tempFileReadCloser{f: f}
153type tempFileReadCloser struct{ f *os.File }
155func (t *tempFileReadCloser) Read(p []byte) (int, error) { return t.f.Read(p) }
157func (t *tempFileReadCloser) Close() error {
158 err := t.f.Close()
159 if rmErr := os.Remove(t.f.Name()); rmErr != nil {
160 dbg.Debugf("dependency firewall: failed to remove upload spill file %s: %v", t.f.Name(), rmErr)
161 }
162 return err
Diagnostics7 git calls · 29 ms
CallStatusSizeTime029 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.3 KiB · hit18 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=18
git.response.bytes=2308
git.response.cache=HIT
git.response.duration_ms=18
git.fetch.wants=61e0653e1e262fc6e2f2e60d95d5648ae8bb7957
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2308
git.pkt.payload.max=2281
git.sideband.band1.bytes=2281
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e280814d070555
Request
e939f8cb…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format