GitLabgitlab-org
cli
main
1package proxy
3import (
4 "archive/zip"
5 "bytes"
6 "encoding/xml"
7 "errors"
8 "io"
9 "mime"
10 "mime/multipart"
11 "net/http"
12 "os"
13 "strings"
15 "gitlab.com/gitlab-org/cli/internal/dbg"
16 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
17)
19// NuGetMatcher recognizes NuGet v3 flatcontainer artifact downloads
20// (".../{id}/{version}/{id}.{version}.nupkg") and pushes: a multipart PUT
21// carrying a .nupkg or .snupkg. Each server advertises its own push URL in
22// its service index, so pushes are recognized by body rather than by path. A
23// .nupkg download outside the flatcontainer shape, such as a nuget.org v2 CDN
24// URL, fails closed.
25//
26// Download coordinates carry the id and version lowercased, as the
27// flatcontainer path spells them; push coordinates keep the casing in the
28// package's nuspec.
29type NuGetMatcher struct {
30 limits peekLimits
31}
33const nupkgSuffix = ".nupkg"
35// maxNuspecSize bounds the decompressed .nuspec read from a pushed package.
36// Manifests are a few KiB; the cap stops a small zip from expanding into a
37// large buffer in the proxy.
38const maxNuspecSize = 1 << 20
40func (m NuGetMatcher) Match(req *http.Request) Match {
41 if req == nil {
42 return Match{}
43 }
44 if req.Method == http.MethodPut {
45 return nugetUpload(req, m.limits.orDefault())
46 }
47 // req.URL.Path is already percent-decoded once by Go's parser. Decoding
48 // again would let a double-encoded request present the matcher a different
49 // path than the upstream NuGet server sees, and because an unmatched
50 // request bypasses the firewall that is a policy-bypass vector. Match
51 // against the once-decoded path the origin actually receives.
52 name, version, result := nugetFlatcontainer(req.URL.Path)
53 switch result {
54 case nugetNotDownload:
55 return Match{}
56 case nugetUnparseable:
57 return unparseableMatch("nuget", policy.Download, "unparseable nuget download URL")
58 }
59 return Match{
60 Matched: true,
61 Pass: true,
62 Coordinate: policy.Coordinate{Ecosystem: "nuget", Name: name, Version: version},
63 Operation: policy.Download,
64 }
65}
67// nugetFlatcontainerResult classifies a request path relative to the NuGet
68// flatcontainer download shape: not a .nupkg download at all, a .nupkg
69// download whose coordinate could not be parsed (fail closed), or a parsed
70// coordinate.
71type nugetFlatcontainerResult int
73const (
74 nugetNotDownload nugetFlatcontainerResult = iota
75 nugetUnparseable
76 nugetParsed
77)
79// nugetFlatcontainer parses ".../{id}/{version}/{id}.{version}.nupkg" and
80// returns the id and version. It anchors on the trailing "<id>.<version>.nupkg"
81// filename matching the two path segments that precede it.
82func nugetFlatcontainer(path string) (string, string, nugetFlatcontainerResult) {
83 trimmed := strings.Trim(path, "/")
84 // The ".nupkg" suffix is the in-scope signal; anything else is not a
85 // NuGet artifact download and passes through untouched.
86 if !strings.HasSuffix(strings.ToLower(trimmed), nupkgSuffix) {
87 return "", "", nugetNotDownload
88 }
89 // A ".nupkg" request whose flatcontainer structure or filename does not
90 // yield a coordinate is recognized-but-unparseable: fail closed rather
91 // than forwarding an in-scope download uninspected.
92 parts := strings.Split(trimmed, "/")
93 if len(parts) < 3 {
94 return "", "", nugetUnparseable
95 }
96 id := parts[len(parts)-3]
97 version := parts[len(parts)-2]
98 file := parts[len(parts)-1]
99 if id == "" || version == "" {
100 return "", "", nugetUnparseable
101 }
102 if !strings.EqualFold(file, id+"."+version+nupkgSuffix) {
103 return "", "", nugetUnparseable
104 }
105 return id, version, nugetParsed
106}
108func nugetUpload(req *http.Request, limits peekLimits) Match {
109 if req.Body == nil {
110 return Match{}
111 }
112 mediaType, params, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
113 if err != nil || !strings.HasPrefix(mediaType, "multipart/") {
114 return Match{}
115 }
116 peeked := peekUploadBody(req.Body, limits)
117 req.Body = peeked.body
118 if peeked.overLimit {
119 return Match{
120 Matched: true,
121 Operation: policy.Upload,
122 Coordinate: policy.Coordinate{Ecosystem: "nuget"},
123 Reason: "upload body too large to inspect for dependency firewall policy",
124 }
125 }
126 // The zip central directory sits at the end of the package, past the
127 // in-memory window for a large push, so read the whole spilled body.
128 src := io.Reader(bytes.NewReader(peeked.data))
129 if peeked.spilled != nil {
130 src = peeked.spilled
131 }
132 id, version, found, err := nugetPackageFromMultipart(src, params["boundary"], limits.inMemory)
133 if err != nil {
134 return unparseableMatch("nuget", policy.Upload, "unparseable publish metadata")
135 }
136 // Every PUT in the session reaches this matcher; a multipart upload with
137 // no package in it, such as a test report, is not a push.
138 if !found {
139 return Match{}
140 }
141 return Match{
142 Matched: true,
143 Pass: true,
144 Coordinate: policy.Coordinate{Ecosystem: "nuget", Name: id, Version: version},
145 Operation: policy.Upload,
146 }
147}
149var errNuGetPackage = errors.New("unparseable nuget package")
151// nugetPackageFromMultipart finds the package among the parts of a push body
152// and reads its coordinate. found is false when no part is a zip archive.
153func nugetPackageFromMultipart(body io.Reader, boundary string, inMemory int64) (string, string, bool, error) {
154 if boundary == "" {
155 return "", "", false, errNuGetPackage
156 }
157 var id, version string
158 found := false
159 mr := multipart.NewReader(body, boundary)
160 for {
161 part, err := mr.NextPart()
162 if errors.Is(err, io.EOF) {
163 return id, version, found, nil
164 }
165 if err != nil {
166 return "", "", false, err
167 }
168 pkg, size, cleanup, err := spoolPart(part, inMemory)
169 if err != nil {
170 return "", "", false, err
171 }
172 partID, partVersion, isPackage, err := nugetPackageCoordinate(pkg, size)
173 cleanup()
174 if err != nil {
175 return "", "", false, err
176 }
177 if !isPackage {
178 continue
179 }
180 // One push carries one package; with two, the one checked need not
181 // be the one the server stores.
182 if found {
183 return "", "", false, errNuGetPackage
184 }
185 id, version, found = partID, partVersion, true
186 }
187}
189// spoolPart buffers a multipart part for the random access archive/zip needs:
190// in memory up to inMemory bytes, otherwise in a temp file that cleanup
191// removes.
192func spoolPart(part io.Reader, inMemory int64) (io.ReaderAt, int64, func(), error) {
193 mem, err := io.ReadAll(io.LimitReader(part, inMemory+1))
194 if err != nil {
195 return nil, 0, nil, err
196 }
197 if int64(len(mem)) <= inMemory {
198 return bytes.NewReader(mem), int64(len(mem)), func() {}, nil
199 }
200 f, err := os.CreateTemp("", "glab-df-nupkg-*")
201 if err != nil {
202 return nil, 0, nil, err
203 }
204 cleanup := func() {
205 _ = f.Close()
206 if err := os.Remove(f.Name()); err != nil {
207 dbg.Debugf("dependency firewall: failed to remove nuget spool file %s: %v", f.Name(), err)
208 }
209 }
210 if _, err := f.Write(mem); err != nil {
211 cleanup()
212 return nil, 0, nil, err
213 }
214 n, err := io.Copy(f, part)
215 if err != nil {
216 cleanup()
217 return nil, 0, nil, err
218 }
219 return f, int64(len(mem)) + n, cleanup, nil
220}
222var zipLocalHeader = []byte("PK\x03\x04")
224// nugetPackageCoordinate reads the coordinate of the package in r. isPackage
225// is false when r is not a zip archive at all.
226func nugetPackageCoordinate(r io.ReaderAt, size int64) (string, string, bool, error) {
227 zr, err := zip.NewReader(r, size)
228 if err != nil {
229 // A part that starts like a zip but does not open is a corrupt
230 // package, not unrelated traffic.
231 head := make([]byte, len(zipLocalHeader))
232 if _, readErr := r.ReadAt(head, 0); readErr == nil && bytes.Equal(head, zipLocalHeader) {
233 return "", "", true, errNuGetPackage
234 }
235 return "", "", false, nil
236 }
237 id, version, err := nuspecCoordinate(zr)
238 return id, version, true, err
239}
241// nuspecCoordinate reads <id> and <version> from the package manifest the way
242// NuGet resolves it: the single .nuspec at the archive root (NuGet.Client
243// PackageHelper.IsManifest), and the one id and version in its metadata.
244func nuspecCoordinate(zr *zip.Reader) (string, string, error) {
245 var manifest *zip.File
246 for _, f := range zr.File {
247 if strings.ContainsAny(f.Name, `/\`) || !strings.HasSuffix(strings.ToLower(f.Name), ".nuspec") {
248 continue
249 }
250 if manifest != nil {
251 return "", "", errNuGetPackage
252 }
253 manifest = f
254 }
255 if manifest == nil {
256 return "", "", errNuGetPackage
257 }
258 rc, err := manifest.Open()
259 if err != nil {
260 return "", "", err
261 }
262 raw, err := io.ReadAll(io.LimitReader(rc, maxNuspecSize+1))
263 if err := errors.Join(err, rc.Close()); err != nil {
264 return "", "", err
265 }
266 if len(raw) > maxNuspecSize {
267 return "", "", errNuGetPackage
268 }
269 var doc struct {
270 Metadata []struct {
271 ID []string `xml:"id"`
272 Version []string `xml:"version"`
273 } `xml:"metadata"`
274 }
275 if err := xml.Unmarshal(raw, &doc); err != nil {
276 return "", "", err
277 }
278 // NuGet reads the first id and version where encoding/xml keeps the last,
279 // so a repeated element could name a coordinate NuGet never uses.
280 if len(doc.Metadata) != 1 || len(doc.Metadata[0].ID) != 1 || len(doc.Metadata[0].Version) != 1 {
281 return "", "", errNuGetPackage
282 }
283 id, version := doc.Metadata[0].ID[0], doc.Metadata[0].Version[0]
284 if id == "" || version == "" {
285 return "", "", errNuGetPackage
286 }
287 return id, version, nil
288}
Diagnostics7 git calls · 233 ms
CallStatusSizeTime0233 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB190 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=190 git.response.bytes=59849 git.response.cache=EXPIRED git.response.duration_ms=190 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:02001.0 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=1051 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1051 git.pkt.payload.max=1024 git.sideband.band1.bytes=1024 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1024 git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598Open pack map
fetch2003.3 KiB · hit34 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=34 git.response.bytes=3374 git.response.cache=HIT git.response.duration_ms=34 git.fetch.wants=92d779360c11db5b344f58ff011a67d0f9464b49 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3374 git.pkt.payload.max=3347 git.sideband.band1.bytes=3347 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e2a0a84e931293
- Request
- e725b28d…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format