GitLabgitlab-org

cli

main
ply7 git calls212 ms99 KiB5 cachedprotocol v2
247 lines9.4 KiBRaw
1//go:build !integration
3package proxy
5import (
6 "archive/zip"
7 "bytes"
8 "io"
9 "mime/multipart"
10 "net/http"
11 "net/http/httptest"
12 "strings"
13 "testing"
15 "github.com/stretchr/testify/assert"
16 "github.com/stretchr/testify/require"
18 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
21func nugetReq(path string) *http.Request {
22 return httptest.NewRequest(http.MethodGet, "https://api.nuget.org/v3-flatcontainer"+path, http.NoBody)
25func TestNuGetMatchDownload(t *testing.T) {
26 t.Parallel()
27 m := NuGetMatcher{}.Match(nugetReq("/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1.nupkg"))
28 assert.True(t, m.Matched)
29 assert.True(t, m.Pass)
30 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "newtonsoft.json", Version: "13.0.1"}, m.Coordinate)
31 assert.Equal(t, policy.Download, m.Operation)
34func TestNuGetMetadataIsNoMatch(t *testing.T) {
35 t.Parallel()
36 m := NuGetMatcher{}.Match(nugetReq("/newtonsoft.json/index.json"))
37 assert.False(t, m.Matched)
40func TestNuGetDoesNotDoubleDecodePath(t *testing.T) {
41 t.Parallel()
42 // req.URL.Path is already decoded once by Go's parser. The matcher must
43 // evaluate that once-decoded path and not decode it a second time. Simulate
44 // a parser output where the file segment still contains a literal "%2e":
45 // upstream sees "...newtonsoft.json.13.0.1%2enupkg" (not a ".nupkg"), so it
46 // must not match. A second PathUnescape would turn "%2enupkg" into ".nupkg"
47 // and wrongly match.
48 req := nugetReq("/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1.nupkg")
49 req.URL.Path = "/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1%2enupkg"
50 m := NuGetMatcher{}.Match(req)
51 assert.False(t, m.Matched, "matcher must not perform a second percent-decode of req.URL.Path")
54func TestNuGetDownloadUnparseableFailsClosed(t *testing.T) {
55 t.Parallel()
56 // A ".nupkg" request whose filename does not match the id/version path
57 // segments is a recognized-but-unparseable download and must fail closed
58 // rather than being forwarded uninspected.
59 m := NuGetMatcher{}.Match(nugetReq("/newtonsoft.json/13.0.1/mismatch.nupkg"))
60 assert.True(t, m.Matched)
61 assert.False(t, m.Pass, "an unparseable nupkg download must fail closed (Pass = false)")
62 assert.Equal(t, "nuget", m.Coordinate.Ecosystem)
63 assert.Equal(t, policy.Download, m.Operation)
66type zipEntry struct{ name, body string }
68func nupkgZip(t *testing.T, entries ...zipEntry) []byte {
69 t.Helper()
70 var buf bytes.Buffer
71 zw := zip.NewWriter(&buf)
72 for _, e := range entries {
73 w, err := zw.Create(e.name)
74 require.NoError(t, err)
75 _, err = w.Write([]byte(e.body))
76 require.NoError(t, err)
77 }
78 require.NoError(t, zw.Close())
79 return buf.Bytes()
82func nuspecXML(id, version string) string {
83 return `<?xml version="1.0"?><package><metadata><id>` + id + `</id><version>` + version + `</version></metadata></package>`
86// nugetPush builds a push request with one multipart file part per entry of
87// parts, the shape dotnet nuget push sends.
88func nugetPush(t *testing.T, parts ...[]byte) (*http.Request, []byte) {
89 t.Helper()
90 var buf bytes.Buffer
91 mw := multipart.NewWriter(&buf)
92 for _, part := range parts {
93 fw, err := mw.CreateFormFile("package", "package.nupkg")
94 require.NoError(t, err)
95 _, err = fw.Write(part)
96 require.NoError(t, err)
97 }
98 require.NoError(t, mw.Close())
99 req := httptest.NewRequest(http.MethodPut, "https://www.nuget.org/api/v2/package", bytes.NewReader(buf.Bytes()))
100 req.Header.Set("Content-Type", mw.FormDataContentType())
101 return req, buf.Bytes()
104func TestNuGetMatchUpload(t *testing.T) {
105 t.Parallel()
106 req, body := nugetPush(t, nupkgZip(t, zipEntry{"Newtonsoft.Json.nuspec", nuspecXML("Newtonsoft.Json", "13.0.1")}))
108 m := NuGetMatcher{}.Match(req)
109 assert.True(t, m.Matched)
110 assert.True(t, m.Pass)
111 assert.Equal(t, policy.Upload, m.Operation)
112 assert.Equal(t, "Newtonsoft.Json", m.Coordinate.Name)
113 assert.Equal(t, "13.0.1", m.Coordinate.Version)
115 restored, err := io.ReadAll(req.Body)
116 require.NoError(t, err)
117 assert.Equal(t, body, restored)
120func TestNuGetUploadReadsPackagedNuspec(t *testing.T) {
121 t.Parallel()
122 // Packed .nupkg files put the nuspec after other entries, start it with a
123 // byte order mark and declare the nuspec XML namespace.
124 nuspec := "\ufeff" + `<?xml version="1.0" encoding="utf-8"?>` +
125 `<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"><metadata minClientVersion="2.12">` +
126 `<id>Newtonsoft.Json</id><version>13.0.1</version><authors>James Newton-King</authors></metadata></package>`
127 req, _ := nugetPush(t, nupkgZip(t,
128 zipEntry{"LICENSE.md", "MIT"},
129 zipEntry{"Newtonsoft.Json.nuspec", nuspec},
130 zipEntry{"lib/net6.0/Newtonsoft.Json.dll", "MZ"},
131 ))
133 m := NuGetMatcher{}.Match(req)
134 assert.True(t, m.Pass)
135 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "Newtonsoft.Json", Version: "13.0.1"}, m.Coordinate)
138func TestNuGetSymbolPackagePushIsChecked(t *testing.T) {
139 t.Parallel()
140 // A .snupkg pushed to the symbol endpoint carries a root nuspec too, so it
141 // is checked under the coordinate it names rather than blocked.
142 req, _ := nugetPush(t, nupkgZip(t,
143 zipEntry{"_rels/.rels", "<Relationships/>"},
144 zipEntry{"Serilog.nuspec", nuspecXML("Serilog", "3.1.1")},
145 zipEntry{"lib/net6.0/Serilog.pdb", "BSJB"},
146 ))
148 m := NuGetMatcher{}.Match(req)
149 assert.True(t, m.Pass)
150 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "Serilog", Version: "3.1.1"}, m.Coordinate)
153func TestNuGetUploadUsesOnlyRootNuspec(t *testing.T) {
154 t.Parallel()
155 // NuGet reads the single .nuspec at the archive root; a nuspec in a
156 // subfolder is package content and must not supply the coordinate.
157 req, _ := nugetPush(t, nupkgZip(t,
158 zipEntry{"content/decoy.nuspec", nuspecXML("Benign", "1.0.0")},
159 zipEntry{"Evil.NUSPEC", nuspecXML("Evil", "6.6.6")},
160 ))
162 m := NuGetMatcher{}.Match(req)
163 assert.True(t, m.Pass)
164 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "Evil", Version: "6.6.6"}, m.Coordinate)
167func TestNuGetUploadAmbiguousPackageFailsClosed(t *testing.T) {
168 t.Parallel()
169 // Each package names a coordinate NuGet would not use, or none at all.
170 repeated := `<package><metadata><id>Benign</id><id>Evil</id><version>1.0.0</version></metadata></package>`
171 cases := map[string][][]byte{
172 "two root nuspecs": {nupkgZip(t, zipEntry{"A.nuspec", nuspecXML("A", "1.0.0")}, zipEntry{"B.nuspec", nuspecXML("B", "2.0.0")})},
173 "only nested": {nupkgZip(t, zipEntry{"content/A.nuspec", nuspecXML("A", "1.0.0")})},
174 "repeated id": {nupkgZip(t, zipEntry{"A.nuspec", repeated})},
175 "missing version": {nupkgZip(t, zipEntry{"A.nuspec", `<package><metadata><id>A</id></metadata></package>`})},
176 "oversized nuspec": {nupkgZip(t, zipEntry{"A.nuspec", nuspecXML("A", "1.0.0") + strings.Repeat(" ", maxNuspecSize)})},
177 "corrupt zip": {[]byte("PK\x03\x04 not really a zip")},
178 "two packages": {nupkgZip(t, zipEntry{"A.nuspec", nuspecXML("A", "1.0.0")}), nupkgZip(t, zipEntry{"B.nuspec", nuspecXML("B", "2.0.0")})},
179 "malformed nuspec": {nupkgZip(t, zipEntry{"A.nuspec", "<package><metadata><id>A"})},
180 "no nuspec entries": {nupkgZip(t, zipEntry{"lib/a.dll", "MZ"})},
181 }
182 for name, parts := range cases {
183 t.Run(name, func(t *testing.T) {
184 t.Parallel()
185 req, _ := nugetPush(t, parts...)
186 m := NuGetMatcher{}.Match(req)
187 assert.True(t, m.Matched)
188 assert.False(t, m.Pass, "an ambiguous or unreadable package must fail closed")
189 assert.Equal(t, "nuget", m.Coordinate.Ecosystem)
190 assert.Equal(t, policy.Upload, m.Operation)
191 })
192 }
195func TestNuGetUploadWithPrependedBytesIsChecked(t *testing.T) {
196 t.Parallel()
197 // Zip readers locate the archive from its end, so bytes in front of a
198 // package do not stop NuGet from reading it; they must not stop the
199 // matcher either.
200 pkg := append([]byte("prefix bytes "), nupkgZip(t, zipEntry{"A.nuspec", nuspecXML("A", "1.0.0")})...)
201 req, _ := nugetPush(t, pkg)
203 m := NuGetMatcher{}.Match(req)
204 assert.True(t, m.Pass)
205 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "A", Version: "1.0.0"}, m.Coordinate)
208func TestNuGetNonPackageMultipartPutIsForwarded(t *testing.T) {
209 t.Parallel()
210 // Every PUT in the session reaches the matcher. A multipart upload with no
211 // package in it, such as a test report, is forwarded untouched.
212 req, body := nugetPush(t, []byte("<testsuite tests=\"1\"/>"))
214 m := NuGetMatcher{}.Match(req)
215 assert.False(t, m.Matched)
217 restored, err := io.ReadAll(req.Body)
218 require.NoError(t, err)
219 assert.Equal(t, body, restored)
222func TestNuGetUploadLargerThanWindowIsChecked(t *testing.T) {
223 t.Parallel()
224 // The zip central directory sits at the end of the package, past the
225 // in-memory window, so the matcher must read the spilled body.
226 pkg := nupkgZip(t, zipEntry{"A.nuspec", nuspecXML("A", "1.0.0")}, zipEntry{"lib/a.dll", strings.Repeat("x", 4096)})
227 req, body := nugetPush(t, pkg)
229 m := NuGetMatcher{limits: peekLimits{inMemory: 64, max: 1 << 20}}.Match(req)
230 assert.True(t, m.Pass)
231 assert.Equal(t, policy.Coordinate{Ecosystem: "nuget", Name: "A", Version: "1.0.0"}, m.Coordinate)
233 restored, err := io.ReadAll(req.Body)
234 require.NoError(t, err)
235 require.NoError(t, req.Body.Close())
236 assert.Equal(t, body, restored, "full body must reach upstream")
239func TestNuGetUploadOverLimitFailsClosed(t *testing.T) {
240 t.Parallel()
241 req, _ := nugetPush(t, nupkgZip(t, zipEntry{"Big.Package.nuspec", nuspecXML("Big.Package", "1.0.0")}))
243 m := NuGetMatcher{limits: peekLimits{inMemory: 8, max: 16}}.Match(req)
244 assert.True(t, m.Matched)
245 assert.False(t, m.Pass, "an over-limit upload must fail closed (Pass = false)")
246 assert.Equal(t, policy.Upload, m.Operation)
Diagnostics7 git calls · 212 ms
CallStatusSizeTime0212 ms
discover200185 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB179 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=169
git.response.bytes=59849
git.response.cache=EXPIRED
git.response.duration_ms=179
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.9 KiB · hit26 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=26
git.response.bytes=2966
git.response.cache=HIT
git.response.duration_ms=26
git.fetch.wants=e96f2971251e3c7b3186c50689035c3e8b51942a
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2966
git.pkt.payload.max=2939
git.sideband.band1.bytes=2939
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e19f024d6a3cd0
Request
ba74f30a…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format