GitLabgitlab-org

cli

main
ply7 git calls329 ms99 KiB5 cachedprotocol v2
156 lines5.3 KiBRaw
1package proxy
3import (
4 "bytes"
5 "io"
6 "net/http"
7 "regexp"
8 "strings"
10 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
13// NPMMatcher recognizes npm artifact tarball downloads and publish uploads
14// against the public npm registry (registry.npmjs.org) or any npm-protocol
15// upstream with the same URL shape.
16type NPMMatcher struct {
17 // limits bounds upload-body inspection. The zero value resolves to the
18 // production defaults; tests set small values to exercise spill/over-limit.
19 limits peekLimits
22func (m NPMMatcher) Match(req *http.Request) Match {
23 if req == nil {
24 return Match{}
25 }
26 if req.Method == http.MethodPut {
27 return npmUpload(req, m.limits.orDefault())
28 }
29 name, version, ok := npmTarball(req.URL.Path)
30 if !ok {
31 return Match{}
32 }
33 return Match{
34 Matched: true,
35 Pass: true,
36 Coordinate: policy.Coordinate{Ecosystem: "npm", Name: name, Version: version},
37 Operation: policy.Download,
38 }
41// npmTarballRegexp matches the npm registry tarball URL shape for both
42// unscoped and scoped packages, anchored on the "/<name>/-/<file>.tgz" suffix
43// so it also matches registries served under a path prefix:
44//
45// /left-pad/-/left-pad-1.3.0.tgz
46// /@babel/core/-/core-7.24.0.tgz
47// /artifactory/api/npm/npm/left-pad/-/left-pad-1.3.0.tgz
48// /api/v4/projects/1/packages/npm/@scope/pkg/-/pkg-1.0.0.tgz
49//
50// The tarball shape ("/-/" separator + ".tgz") is the reliable enforcement
51// signal; the leading path segments vary by registry (public npm, Artifactory,
52// GitLab's own package registry). Anchoring on "^/?<name>" instead would
53// silently skip every path-prefixed registry — those fetches would be
54// forwarded with no policy check and no verdict, uninspected but green — so
55// the leading "(?:^|/)" lets any prefix precede the coordinate.
56//
57// Named groups:
58//
59// name — full package name including optional @scope/
60// short — bare package name without scope (used in the filename)
61// file — filename base "<short>-<version>"
62//
63// Go's RE2 engine has no backreferences, so the regexp cannot assert that
64// the filename repeats the short name; npmTarball verifies that "file"
65// starts with "short-" and derives the version from the remainder.
66var npmTarballRegexp = regexp.MustCompile(
67 `(?:^|/)(?P<name>(?:@[^/]+/)?(?P<short>[^/]+))/-/(?P<file>[^/]+)\.tgz$`,
70// npmTarball parses "/<name>/-/<file>-<version>.tgz" (scoped:
71// "/@scope/name/-/name-<version>.tgz").
72func npmTarball(path string) (string, string, bool) {
73 m := npmTarballRegexp.FindStringSubmatch(path)
74 if m == nil {
75 return "", "", false
76 }
77 name := m[npmTarballRegexp.SubexpIndex("name")]
78 short := m[npmTarballRegexp.SubexpIndex("short")]
79 file := m[npmTarballRegexp.SubexpIndex("file")]
80 if name == "" || short == "" {
81 return "", "", false
82 }
83 version, ok := strings.CutPrefix(file, short+"-")
84 if !ok || version == "" {
85 return "", "", false
86 }
87 return name, version, true
90// npmUpload reads a publish body to extract the name and the single version
91// being published. It restores req.Body. A body too large to inspect (see
92// peekLimits.max) fails closed by leaving Pass false.
93func npmUpload(req *http.Request, limits peekLimits) Match {
94 if req.Body == nil {
95 return Match{}
96 }
97 peeked := peekUploadBody(req.Body, limits)
98 req.Body = peeked.body
99 if peeked.overLimit {
100 return Match{
101 Matched: true,
102 Operation: policy.Upload,
103 Coordinate: policy.Coordinate{Ecosystem: "npm"},
104 Reason: "upload body too large to inspect for dependency firewall policy",
105 }
106 }
108 // Every PUT in the session reaches this matcher, including npm's metadata
109 // writes (dist-tags, owners, stars) and non-registry uploads. A body that
110 // is not a JSON object cannot be a packument, so it is forwarded.
111 if !bytes.HasPrefix(bytes.TrimLeft(peeked.data, " \t\r\n"), []byte("{")) {
112 return Match{}
113 }
114 // A tarball larger than the in-memory window cuts the packument off in
115 // peeked.data, so scan the whole spilled body instead.
116 src := io.Reader(bytes.NewReader(peeked.data))
117 if peeked.spilled != nil {
118 src = peeked.spilled
119 }
120 doc, err := scanPackument(src, int(limits.inMemory))
121 // An object we cannot decode may be a publish: fail closed rather than
122 // forwarding it uninspected.
123 if err != nil {
124 return unparseableMatch("npm", policy.Upload, "unparseable publish metadata")
125 }
126 // Only a publish carries the tarball in _attachments; metadata writes such
127 // as deprecate or owner changes do not, and are forwarded.
128 if !doc.attachments {
129 return Match{}
130 }
131 // A publish carries exactly one version. With several, the one checked
132 // need not be the one the registry stores.
133 if doc.name == "" || len(doc.versions) != 1 {
134 return unparseableMatch("npm", policy.Upload, "unparseable publish metadata")
135 }
136 return Match{
137 Matched: true,
138 Pass: true,
139 Coordinate: policy.Coordinate{Ecosystem: "npm", Name: doc.name, Version: doc.versions[0]},
140 Operation: policy.Upload,
141 }
144// bodyWithPrefix returns a ReadCloser yielding prefix then the remaining
145// bytes of rest, closing rest when done. Upload matchers use it to peek at
146// the leading bytes of a request body and then restore the full stream for
147// the upstream round trip.
148func bodyWithPrefix(prefix []byte, rest io.ReadCloser) io.ReadCloser {
149 return struct {
150 io.Reader
151 io.Closer
152 }{
153 Reader: io.MultiReader(bytes.NewReader(prefix), rest),
154 Closer: rest,
155 }
Diagnostics7 git calls · 329 ms
CallStatusSizeTime0329 ms
discover200185 B4 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=317
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.3 KiB317 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=2355
git.response.cache=MISS
git.response.duration_ms=317
git.fetch.wants=fc306576729a217a463deb1f1b7d65a891f6848f
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2355
git.pkt.payload.max=2328
git.sideband.band1.bytes=2328
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e25090cffc0ecd
Request
9bfe8c52…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format