GitLabgitlab-org

cli

main
ply7 git calls258 ms99 KiB5 cachedprotocol v2
181 lines6.2 KiBRaw
1package proxy
3import (
4 "encoding/binary"
5 "encoding/json"
6 "net/http"
7 "strings"
9 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
12// CargoMatcher recognizes crate downloads and publishes. Downloads take two
13// shapes: the legacy registry API path
14// ("/api/v1/crates/{crate}/{version}/download") and the sparse-index
15// download of a "{crate}-{version}.crate" file (default since cargo 1.70).
16// The sparse download URL is defined by the registry's config.json "dl"
17// template and so varies by registry (crates.io serves
18// "static.crates.io/crates/{crate}/{crate}-{version}.crate"; third parties
19// such as Artifactory and Cloudsmith use their own path layouts), but the
20// requested file is always named "{crate}-{version}.crate". Matching on that
21// ".crate" filename keeps the matcher working behind third-party registries.
22//
23// Publishes are PUT "/api/v1/crates/new". The publish body is a
24// length-prefixed frame: [u32 LE json_len][json][u32 LE crate_len][.crate];
25// the JSON carries the name and vers.
26type CargoMatcher struct {
27 limits peekLimits
30func (m CargoMatcher) Match(req *http.Request) Match {
31 if req == nil {
32 return Match{}
33 }
34 if req.Method == http.MethodPut {
35 return cargoPublish(req, m.limits.orDefault())
36 }
37 name, version, result := cargoDownload(req.URL.Path)
38 switch result {
39 case cargoNotDownload:
40 return Match{}
41 case cargoUnparseable:
42 return unparseableMatch("cargo", policy.Download, "unparseable crate download URL")
43 }
44 return Match{
45 Matched: true,
46 Pass: true,
47 Coordinate: policy.Coordinate{Ecosystem: "cargo", Name: name, Version: version},
48 Operation: policy.Download,
49 }
52// cargoDownloadResult classifies a request path relative to the crate
53// download shapes: not a download at all, a recognized download shape whose
54// coordinate could not be parsed (fail closed), or a parsed coordinate.
55type cargoDownloadResult int
57const (
58 cargoNotDownload cargoDownloadResult = iota
59 cargoUnparseable
60 cargoParsed
63// cargoDownload parses a crate download URL. It recognizes two shapes:
64//
65// - The "download" endpoint "…/crates/{crate}/{version}/download". This
66// covers both the legacy registry API path
67// "/api/v1/crates/{crate}/{version}/download" and the crates.io sparse
68// form "static.crates.io/crates/{crate}/{version}/download" that cargo
69// builds when the registry's config.json "dl" template has no markers
70// (crates.io's does not), so the "/{crate}/{version}/download" suffix is
71// appended to the "…/crates" base. Anchoring on the "crates" segment and
72// the "download" suffix — rather than a fixed "/api/v1" prefix — matches
73// both without over-fitting to one registry's layout.
74// - The sparse-index download of a "{crate}-{version}.crate" file, used by
75// registries whose "dl" template includes the "{crate}"/"{version}"
76// markers (for example Cloudsmith).
77func cargoDownload(path string) (string, string, cargoDownloadResult) {
78 if strings.HasSuffix(path, ".crate") {
79 return cargoCrateFile(path)
80 }
81 trimmed := strings.Trim(path, "/")
82 parts := strings.Split(trimmed, "/")
83 // … crates {crate} {version} download
84 if len(parts) < 4 {
85 return "", "", cargoNotDownload
86 }
87 tail := parts[len(parts)-4:]
88 if tail[0] != "crates" || tail[3] != "download" {
89 return "", "", cargoNotDownload
90 }
91 // The "…/crates/{crate}/{version}/download" shape is a recognized crate
92 // download; empty name/version segments make it unparseable, so fail
93 // closed rather than forwarding an in-scope download uninspected.
94 if tail[1] == "" || tail[2] == "" {
95 return "", "", cargoUnparseable
96 }
97 return tail[1], tail[2], cargoParsed
100// cargoCrateFile extracts the coordinate from a "{crate}-{version}.crate"
101// filename at the end of a download path. A crate name may contain "-" (for
102// example "serde-derive") and a version is a semver value that starts with a
103// digit, so the name/version boundary is the first "-" immediately followed
104// by a digit.
105func cargoCrateFile(path string) (string, string, cargoDownloadResult) {
106 filename := path
107 if _, after, ok := strings.CutLast(path, "/"); ok {
108 filename = after
109 }
110 base := strings.TrimSuffix(filename, ".crate")
111 for i := 0; i+1 < len(base); i++ {
112 if base[i] != '-' {
113 continue
114 }
115 if next := base[i+1]; next < '0' || next > '9' {
116 continue
117 }
118 name := base[:i]
119 version := base[i+1:]
120 if name == "" || version == "" {
121 return "", "", cargoUnparseable
122 }
123 return name, version, cargoParsed
124 }
125 // The ".crate" suffix already marks this as a crate download; a filename
126 // this heuristic cannot split into name/version is a recognized-but-
127 // unparseable download, so fail closed instead of forwarding it.
128 return "", "", cargoUnparseable
131func cargoPublish(req *http.Request, limits peekLimits) Match {
132 if req.Body == nil {
133 return Match{}
134 }
135 path := strings.Trim(req.URL.Path, "/")
136 if path != "api/v1/crates/new" && !strings.HasSuffix(path, "/api/v1/crates/new") {
137 return Match{}
138 }
139 peeked := peekUploadBody(req.Body, limits)
140 req.Body = peeked.body
141 if peeked.overLimit {
142 return Match{
143 Matched: true,
144 Operation: policy.Upload,
145 Coordinate: policy.Coordinate{Ecosystem: "cargo"},
146 Reason: "upload body too large to inspect for dependency firewall policy",
147 }
148 }
149 name, version, ok := cargoPublishMeta(peeked.data)
150 if !ok {
151 // The request already matched "PUT …/api/v1/crates/new" exactly, so
152 // it is definitely a publish; a frame we cannot decode fails closed
153 // rather than falling out of classification.
154 return unparseableMatch("cargo", policy.Upload, "unparseable publish metadata")
155 }
156 return Match{
157 Matched: true,
158 Pass: true,
159 Coordinate: policy.Coordinate{Ecosystem: "cargo", Name: name, Version: version},
160 Operation: policy.Upload,
161 }
164// cargoPublishMeta reads the leading [u32 LE json_len][json] frame.
165func cargoPublishMeta(data []byte) (string, string, bool) {
166 if len(data) < 4 {
167 return "", "", false
168 }
169 jsonLen := binary.LittleEndian.Uint32(data[:4])
170 if jsonLen == 0 || uint64(jsonLen) > uint64(len(data)-4) {
171 return "", "", false
172 }
173 var meta struct {
174 Name string `json:"name"`
175 Vers string `json:"vers"`
176 }
177 if err := json.Unmarshal(data[4:4+jsonLen], &meta); err != nil || meta.Name == "" {
178 return "", "", false
179 }
180 return meta.Name, meta.Vers, true
Diagnostics7 git calls · 258 ms
CallStatusSizeTime0258 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=5
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit5 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=245
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=5
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.4 KiB245 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=2408
git.response.cache=MISS
git.response.duration_ms=245
git.fetch.wants=e63ca49c253adaacf23e31f34d74fad89989f979
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2408
git.pkt.payload.max=2381
git.sideband.band1.bytes=2381
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e22c7f79b92a83
Request
41b646d8…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format