GitLabgitlab-org

cli

main
ply7 git calls43 ms99 KiB6 cachedprotocol v2
164 lines5.2 KiBRaw
1package proxy
3import (
4 "bytes"
5 "io"
6 "mime"
7 "mime/multipart"
8 "net/http"
9 "regexp"
10 "strings"
12 "gitlab.com/gitlab-org/cli/internal/dbg"
13 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
16// PyPIMatcher recognizes PyPI artifact downloads (wheels/sdists served
17// from files.pythonhosted.org) and twine uploads (POST multipart to
18// upload.pypi.org/legacy/).
19type PyPIMatcher struct {
20 // limits bounds upload-body inspection. The zero value resolves to the
21 // production defaults; tests set small values to exercise spill/over-limit.
22 limits peekLimits
25func (m PyPIMatcher) Match(req *http.Request) Match {
26 if req == nil {
27 return Match{}
28 }
29 if req.Method == http.MethodPost {
30 return pypiUpload(req, m.limits.orDefault())
31 }
32 name, version, ok := pypiFile(req.URL.Path)
33 if !ok {
34 return Match{}
35 }
36 return Match{
37 Matched: true,
38 Pass: true,
39 Coordinate: policy.Coordinate{Ecosystem: "pypi", Name: name, Version: version},
40 Operation: policy.Download,
41 }
44// sdistExts are the source-distribution archive suffixes PyPI serves. Modern
45// uploads are .tar.gz or .zip (PEP 527), but PyPI still serves the legacy
46// .tar.bz2 and .tgz sdists for older packages; all carry the same
47// "<name>-<version>" filename shape and are split identically. Recognizing
48// every served format matters because an unmatched download is forwarded
49// without a policy check (fail-open), so a missing extension is a bypass.
50var sdistExts = []string{".tar.gz", ".zip", ".tar.bz2", ".tgz"}
52// pypiFile parses a wheel or sdist filename at the end of a download path.
53// Only .whl (wheel) and the sdistExts source archives are recognized; other
54// formats degrade to no match.
55func pypiFile(path string) (string, string, bool) {
56 filename := path
57 if _, after, ok := strings.CutLast(path, "/"); ok {
58 filename = after
59 }
60 // A PEP 658 sidecar (<wheel>.whl.metadata) is often the only request pip
61 // makes for a blocked package; treat it as the wheel it describes.
62 filename = strings.TrimSuffix(filename, ".metadata")
63 if base, ok := strings.CutSuffix(filename, ".whl"); ok {
64 fields := strings.Split(base, "-")
65 if len(fields) < 2 {
66 return "", "", false
67 }
68 return normalizePyPIName(fields[0]), fields[1], true
69 }
70 for _, ext := range sdistExts {
71 base, ok := strings.CutSuffix(filename, ext)
72 if !ok {
73 continue
74 }
75 i := strings.LastIndex(base, "-")
76 if i <= 0 {
77 return "", "", false
78 }
79 return normalizePyPIName(base[:i]), base[i+1:], true
80 }
81 return "", "", false
84func pypiUpload(req *http.Request, limits peekLimits) Match {
85 if req.Body == nil {
86 return Match{}
87 }
88 // twine uploads target the PyPI legacy endpoint (upload.pypi.org/legacy/).
89 // Anchor on that path so unrelated multipart POST traffic passing through
90 // the proxy is not misclassified as a package upload, the way GemMatcher
91 // anchors on "api/v1/gems" and MavenMatcher on the artifact path shape.
92 if !strings.HasSuffix(strings.TrimRight(req.URL.Path, "/"), "/legacy") {
93 return Match{}
94 }
95 mediaType, params, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
96 if err != nil || !strings.HasPrefix(mediaType, "multipart/") {
97 return Match{}
98 }
99 peeked := peekUploadBody(req.Body, limits)
100 req.Body = peeked.body
101 if peeked.overLimit {
102 return Match{
103 Matched: true,
104 Operation: policy.Upload,
105 Coordinate: policy.Coordinate{Ecosystem: "pypi"},
106 Reason: "upload body too large to inspect for dependency firewall policy",
107 }
108 }
109 name, version := pypiFieldsFromMultipart(peeked.data, params["boundary"])
110 if name == "" {
111 // The request already matched the "/legacy" endpoint with a multipart
112 // body, so it is definitely a twine upload; a body we cannot parse a
113 // name out of fails closed rather than forwarding it uninspected.
114 return unparseableMatch("pypi", policy.Upload, "unparseable publish metadata")
115 }
116 return Match{
117 Matched: true,
118 Pass: true,
119 Coordinate: policy.Coordinate{Ecosystem: "pypi", Name: normalizePyPIName(name), Version: version},
120 Operation: policy.Upload,
121 }
124var pypiNameSeparators = regexp.MustCompile(`[-_.]+`)
126// normalizePyPIName applies PEP 503 normalization: lowercase and collapse
127// any run of "-", "_", or "." into a single "-".
128func normalizePyPIName(name string) string {
129 return pypiNameSeparators.ReplaceAllString(strings.ToLower(name), "-")
132// pypiFieldsFromMultipart returns the "name" and "version" form fields from
133// a twine multipart upload body. It reads only the small text fields and
134// skips the file content part to avoid buffering large uploads.
135func pypiFieldsFromMultipart(body []byte, boundary string) (string, string) {
136 if boundary == "" {
137 return "", ""
138 }
139 mr := multipart.NewReader(bytes.NewReader(body), boundary)
140 var name, version string
141 for {
142 part, err := mr.NextPart()
143 if err != nil {
144 break
145 }
146 field := part.FormName()
147 if field != "name" && field != "version" {
148 _ = part.Close()
149 continue
150 }
151 val, err := io.ReadAll(io.LimitReader(part, 1024))
152 if err != nil {
153 dbg.Debugf("dependency firewall pypi matcher: failed to read multipart field %q: %v", field, err)
154 }
155 _ = part.Close()
156 switch field {
157 case "name":
158 name = strings.TrimSpace(string(val))
159 case "version":
160 version = strings.TrimSpace(string(val))
161 }
162 }
163 return name, version
Diagnostics7 git calls · 43 ms
CallStatusSizeTime043 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit5 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=5
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit7 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=7
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=7
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.2 KiB · hit25 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=25
git.response.bytes=2276
git.response.cache=HIT
git.response.duration_ms=25
git.fetch.wants=fd1e6ea06e2e188f7233adce881468a21bde9227
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2276
git.pkt.payload.max=2249
git.sideband.band1.bytes=2249
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e174cddad43488
Request
cd419450…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format