GitLabgitlab-org
cli
main
1package proxy
3import (
4 "encoding/binary"
5 "encoding/json"
6 "net/http"
7 "strings"
9 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
10)
12// CargoMatcher recognizes crate downloads and publishes. Downloads take two
13// shapes: the legacy registry API path
14// ("/api/v1/crates/{crate}/{version}/download") and the sparse-index
15// download of a "{crate}-{version}.crate" file (default since cargo 1.70).
16// The sparse download URL is defined by the registry's config.json "dl"
17// template and so varies by registry (crates.io serves
18// "static.crates.io/crates/{crate}/{crate}-{version}.crate"; third parties
19// such as Artifactory and Cloudsmith use their own path layouts), but the
20// requested file is always named "{crate}-{version}.crate". Matching on that
21// ".crate" filename keeps the matcher working behind third-party registries.
22//
23// Publishes are PUT "/api/v1/crates/new". The publish body is a
24// length-prefixed frame: [u32 LE json_len][json][u32 LE crate_len][.crate];
25// the JSON carries the name and vers.
26type CargoMatcher struct {
27 limits peekLimits
28}
30func (m CargoMatcher) Match(req *http.Request) Match {
31 if req == nil {
32 return Match{}
33 }
34 if req.Method == http.MethodPut {
35 return cargoPublish(req, m.limits.orDefault())
36 }
37 name, version, result := cargoDownload(req.URL.Path)
38 switch result {
39 case cargoNotDownload:
40 return Match{}
41 case cargoUnparseable:
42 return unparseableMatch("cargo", policy.Download, "unparseable crate download URL")
43 }
44 return Match{
45 Matched: true,
46 Pass: true,
47 Coordinate: policy.Coordinate{Ecosystem: "cargo", Name: name, Version: version},
48 Operation: policy.Download,
49 }
50}
52// cargoDownloadResult classifies a request path relative to the crate
53// download shapes: not a download at all, a recognized download shape whose
54// coordinate could not be parsed (fail closed), or a parsed coordinate.
55type cargoDownloadResult int
57const (
58 cargoNotDownload cargoDownloadResult = iota
59 cargoUnparseable
60 cargoParsed
61)
63// cargoDownload parses a crate download URL. It recognizes two shapes:
64//
65// - The "download" endpoint "…/crates/{crate}/{version}/download". This
66// covers both the legacy registry API path
67// "/api/v1/crates/{crate}/{version}/download" and the crates.io sparse
68// form "static.crates.io/crates/{crate}/{version}/download" that cargo
69// builds when the registry's config.json "dl" template has no markers
70// (crates.io's does not), so the "/{crate}/{version}/download" suffix is
71// appended to the "…/crates" base. Anchoring on the "crates" segment and
72// the "download" suffix — rather than a fixed "/api/v1" prefix — matches
73// both without over-fitting to one registry's layout.
74// - The sparse-index download of a "{crate}-{version}.crate" file, used by
75// registries whose "dl" template includes the "{crate}"/"{version}"
76// markers (for example Cloudsmith).
77func cargoDownload(path string) (string, string, cargoDownloadResult) {
78 if strings.HasSuffix(path, ".crate") {
79 return cargoCrateFile(path)
80 }
81 trimmed := strings.Trim(path, "/")
82 parts := strings.Split(trimmed, "/")
83 // … crates {crate} {version} download
84 if len(parts) < 4 {
85 return "", "", cargoNotDownload
86 }
87 tail := parts[len(parts)-4:]
88 if tail[0] != "crates" || tail[3] != "download" {
89 return "", "", cargoNotDownload
90 }
91 // The "…/crates/{crate}/{version}/download" shape is a recognized crate
92 // download; empty name/version segments make it unparseable, so fail
93 // closed rather than forwarding an in-scope download uninspected.
94 if tail[1] == "" || tail[2] == "" {
95 return "", "", cargoUnparseable
96 }
97 return tail[1], tail[2], cargoParsed
98}
100// cargoCrateFile extracts the coordinate from a "{crate}-{version}.crate"
101// filename at the end of a download path. A crate name may contain "-" (for
102// example "serde-derive") and a version is a semver value that starts with a
103// digit, so the name/version boundary is the first "-" immediately followed
104// by a digit.
105func cargoCrateFile(path string) (string, string, cargoDownloadResult) {
106 filename := path
107 if _, after, ok := strings.CutLast(path, "/"); ok {
108 filename = after
109 }
110 base := strings.TrimSuffix(filename, ".crate")
111 for i := 0; i+1 < len(base); i++ {
112 if base[i] != '-' {
113 continue
114 }
115 if next := base[i+1]; next < '0' || next > '9' {
116 continue
117 }
118 name := base[:i]
119 version := base[i+1:]
120 if name == "" || version == "" {
121 return "", "", cargoUnparseable
122 }
123 return name, version, cargoParsed
124 }
125 // The ".crate" suffix already marks this as a crate download; a filename
126 // this heuristic cannot split into name/version is a recognized-but-
127 // unparseable download, so fail closed instead of forwarding it.
128 return "", "", cargoUnparseable
129}
131func cargoPublish(req *http.Request, limits peekLimits) Match {
132 if req.Body == nil {
133 return Match{}
134 }
135 path := strings.Trim(req.URL.Path, "/")
136 if path != "api/v1/crates/new" && !strings.HasSuffix(path, "/api/v1/crates/new") {
137 return Match{}
138 }
139 peeked := peekUploadBody(req.Body, limits)
140 req.Body = peeked.body
141 if peeked.overLimit {
142 return Match{
143 Matched: true,
144 Operation: policy.Upload,
145 Coordinate: policy.Coordinate{Ecosystem: "cargo"},
146 Reason: "upload body too large to inspect for dependency firewall policy",
147 }
148 }
149 name, version, ok := cargoPublishMeta(peeked.data)
150 if !ok {
151 // The request already matched "PUT …/api/v1/crates/new" exactly, so
152 // it is definitely a publish; a frame we cannot decode fails closed
153 // rather than falling out of classification.
154 return unparseableMatch("cargo", policy.Upload, "unparseable publish metadata")
155 }
156 return Match{
157 Matched: true,
158 Pass: true,
159 Coordinate: policy.Coordinate{Ecosystem: "cargo", Name: name, Version: version},
160 Operation: policy.Upload,
161 }
162}
164// cargoPublishMeta reads the leading [u32 LE json_len][json] frame.
165func cargoPublishMeta(data []byte) (string, string, bool) {
166 if len(data) < 4 {
167 return "", "", false
168 }
169 jsonLen := binary.LittleEndian.Uint32(data[:4])
170 if jsonLen == 0 || uint64(jsonLen) > uint64(len(data)-4) {
171 return "", "", false
172 }
173 var meta struct {
174 Name string `json:"name"`
175 Vers string `json:"vers"`
176 }
177 if err := json.Unmarshal(data[4:4+jsonLen], &meta); err != nil || meta.Name == "" {
178 return "", "", false
179 }
180 return meta.Name, meta.Vers, true
181}
Diagnostics7 git calls · 258 ms
CallStatusSizeTime0258 ms
discover200185 B2 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=59849 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=5 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:02001.0 KiB · hit5 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=245 git.response.bytes=1051 git.response.cache=HIT git.response.duration_ms=5 git.fetch.filter=tree:0 git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1051 git.pkt.payload.max=1024 git.sideband.band1.bytes=1024 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1024 git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598Open pack map
fetch2002.4 KiB245 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2408 git.response.cache=MISS git.response.duration_ms=245 git.fetch.wants=e63ca49c253adaacf23e31f34d74fad89989f979 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2408 git.pkt.payload.max=2381 git.sideband.band1.bytes=2381 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e22c7f79b92a83
- Request
- 41b646d8…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format