GitLabgitlab-org

cli

main
ply7 git calls37 ms99 KiB5 cachedprotocol v2
374 lines11 KiBRaw
1//go:build !integration
3package gomodproxy
5import (
6 "context"
7 "io"
8 "net/http"
9 "slices"
10 "strings"
11 "sync"
12 "sync/atomic"
13 "testing"
14 "time"
16 "github.com/goproxy/goproxy"
17 "github.com/stretchr/testify/assert"
18 "github.com/stretchr/testify/require"
20 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
21 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/verdict"
24type coordChecker struct {
25 mu sync.Mutex
26 asked []policy.Request
27 blocked policy.Coordinate
30func (c *coordChecker) Check(_ context.Context, req policy.Request) (policy.Result, error) {
31 c.mu.Lock()
32 c.asked = append(c.asked, req)
33 c.mu.Unlock()
34 if req.Coordinate == c.blocked {
35 return policy.Result{Verdict: verdict.Blocked, Reason: "blocked by policy"}, nil
36 }
37 return policy.Result{}, nil
40func (c *coordChecker) requests() []policy.Request {
41 c.mu.Lock()
42 defer c.mu.Unlock()
43 return slices.Clone(c.asked)
46func (c *coordChecker) coordinates() []policy.Coordinate {
47 var coords []policy.Coordinate
48 for _, r := range c.requests() {
49 coords = append(coords, r.Coordinate)
50 }
51 return coords
54func TestZipCheckSendsFullPolicyRequest(t *testing.T) {
55 t.Parallel()
56 checker := &coordChecker{}
57 s := startServer(t, checker, &backendStub{})
59 res := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
61 assert.Equal(t, http.StatusOK, res.status)
62 assert.Equal(t, []policy.Request{{
63 Coordinate: policy.Coordinate{Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1"},
64 ProjectID: "group/proj",
65 Operation: policy.Download,
66 }}, checker.requests())
69func TestPercentEncodedVersionCannotBypassGate(t *testing.T) {
70 t.Parallel()
71 checker := &coordChecker{blocked: policy.Coordinate{
72 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
73 }}
74 backend := &backendStub{body: "BLOCKED-MODULE-BYTES"}
75 s := startServer(t, checker, backend)
77 plain := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
78 require.Equal(t, http.StatusForbidden, plain.status, "the unencoded form must be blocked")
80 encoded := get(t, s, "/github.com/pkg/errors/@v/v0%2e9%2e1.zip")
81 assert.Equal(t, http.StatusNotFound, encoded.status)
82 assert.NotContains(t, encoded.body, "BLOCKED-MODULE-BYTES")
83 assert.Equal(t, 0, backend.count(), "a percent-encoded path must never reach the backend")
86func TestPercentEncodedPathsAreRefused(t *testing.T) {
87 t.Parallel()
88 checker := &coordChecker{}
89 backend := &backendStub{}
90 s := startServer(t, checker, backend)
92 for _, p := range []string{
93 "/sumdb/%2e%2e/evil.com/@v/v1.0.0.zip",
94 "/%2e%2e/evil.com/@v/v1.0.0.zip",
95 "/evil.com/@v/..%2fv1.0.0.zip",
96 "/evil.com/@v/v1%2e0%2e0.zip",
97 } {
98 res := get(t, s, p)
99 assert.Equal(t, http.StatusNotFound, res.status, p)
100 }
101 assert.Zero(t, backend.count(), "no encoded path may reach the backend")
102 assert.Empty(t, checker.coordinates(), "no encoded path may reach the firewall")
105func TestCaseEncodedModulePathIsGated(t *testing.T) {
106 t.Parallel()
107 checker := &coordChecker{blocked: policy.Coordinate{
108 Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.5.0",
109 }}
110 backend := &backendStub{body: "MODULE-BYTES"}
111 s := startServer(t, checker, backend)
113 info := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.6.0.info")
114 assert.Equal(t, http.StatusOK, info.status)
116 zip := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.6.0.zip")
117 assert.Equal(t, http.StatusOK, zip.status)
118 assert.Equal(t, "MODULE-BYTES", zip.body)
120 blocked := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.5.0.zip")
121 assert.Equal(t, http.StatusForbidden, blocked.status)
122 assert.NotContains(t, blocked.body, "MODULE-BYTES")
124 assert.Equal(t, []policy.Coordinate{
125 {Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.6.0"},
126 {Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.5.0"},
127 }, checker.coordinates())
128 assert.Equal(t, []string{
129 "/github.com/!burnt!sushi/toml/@v/v1.6.0.info",
130 "/github.com/!burnt!sushi/toml/@v/v1.6.0.zip",
131 }, backend.paths, "the backend must read the '!' form the gate checked")
134func TestCaseEncodingDoesNotReopenPercentBypass(t *testing.T) {
135 t.Parallel()
136 checker := &coordChecker{}
137 backend := &backendStub{}
138 s := startServer(t, checker, backend)
140 for _, p := range []string{
141 "/github.com/%21burnt%21sushi/toml/@v/v1%2e6%2e0.zip",
142 "/github.com/%21burnt%21sushi/%2e%2e/evil.com/@v/v1.0.0.zip",
143 "/github.com/%2521burnt/toml/@v/v1.6.0.zip",
144 } {
145 res := get(t, s, p)
146 assert.Equal(t, http.StatusNotFound, res.status, p)
147 }
148 assert.Zero(t, backend.count(), "no other encoding may reach the backend")
149 assert.Empty(t, checker.coordinates(), "no other encoding may reach the firewall")
152type nopSeekCloser struct{ *strings.Reader }
154func (nopSeekCloser) Close() error { return nil }
156type fixtureFetcher struct{ zip string }
158func (f fixtureFetcher) Query(_ context.Context, _, _ string) (string, time.Time, error) {
159 return "v0.9.1", time.Unix(0, 0), nil
162func (f fixtureFetcher) List(_ context.Context, _ string) ([]string, error) {
163 return []string{"v0.9.1"}, nil
166func (f fixtureFetcher) Download(_ context.Context, _, version string) (io.ReadSeekCloser, io.ReadSeekCloser, io.ReadSeekCloser, error) {
167 return nopSeekCloser{strings.NewReader(`{"Version":"` + version + `","Time":"1970-01-01T00:00:00Z"}`)},
168 nopSeekCloser{strings.NewReader("module github.com/pkg/errors\n")},
169 nopSeekCloser{strings.NewReader(f.zip)},
170 nil
173func TestGateAgreesWithRealBackend(t *testing.T) {
174 t.Parallel()
175 checker := &coordChecker{blocked: policy.Coordinate{
176 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
177 }}
178 backend := &goproxy.Goproxy{Fetcher: fixtureFetcher{zip: "REAL-ZIP"}}
179 s := startServer(t, checker, backend)
181 blocked := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
182 assert.Equal(t, http.StatusForbidden, blocked.status)
183 assert.NotContains(t, blocked.body, "REAL-ZIP")
185 encoded := get(t, s, "/github.com/pkg/errors/@v/v0%2e9%2e1.zip")
186 assert.Equal(t, http.StatusNotFound, encoded.status)
187 assert.NotContains(t, encoded.body, "REAL-ZIP")
189 assert.Equal(t, []policy.Coordinate{
190 {Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1"},
191 }, checker.coordinates(), "gate and backend must agree on the coordinate")
194func TestAllowedZipFlowsThroughRealBackend(t *testing.T) {
195 t.Parallel()
196 checker := &coordChecker{}
197 backend := &goproxy.Goproxy{Fetcher: fixtureFetcher{zip: "REAL-ZIP"}}
198 s := startServer(t, checker, backend)
200 res := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
201 assert.Equal(t, http.StatusOK, res.status)
202 assert.Equal(t, "REAL-ZIP", res.body)
204 info := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.info")
205 assert.Equal(t, http.StatusOK, info.status)
206 assert.Contains(t, info.body, "v0.9.1")
209type countingHandler struct {
210 next http.Handler
211 hits atomic.Int64
214func (c *countingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
215 c.hits.Add(1)
216 c.next.ServeHTTP(w, r)
219// Serving the .mod makes goproxy fetch and cache the zip too, so the gate,
220// not the cache, must refuse the zip.
221func TestBlockedZipRefusedAfterBackendCachedModule(t *testing.T) {
222 t.Parallel()
223 checker := &coordChecker{blocked: policy.Coordinate{
224 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
225 }}
226 backend := &countingHandler{next: &goproxy.Goproxy{
227 Fetcher: fixtureFetcher{zip: "REAL-ZIP"},
228 Cacher: goproxy.DirCacher(t.TempDir()),
229 }}
230 s := startServer(t, checker, backend)
232 mod := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.mod")
233 require.Equal(t, http.StatusOK, mod.status)
235 zip := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
236 assert.Equal(t, http.StatusForbidden, zip.status)
237 assert.NotContains(t, zip.body, "REAL-ZIP")
238 assert.Equal(t, int64(1), backend.hits.Load(), "only the .mod reaches the backend")
241func TestGateChecksUnescapedVersion(t *testing.T) {
242 t.Parallel()
243 checker := &coordChecker{}
244 s := startServer(t, checker, &backendStub{})
246 res := get(t, s, "/example.com/m/@v/v1.0.0-!r!c1.zip")
248 assert.Equal(t, http.StatusOK, res.status)
249 assert.Equal(t, []policy.Coordinate{
250 {Ecosystem: "golang", Name: "example.com/m", Version: "v1.0.0-RC1"},
251 }, checker.coordinates())
254func assertRefused(t *testing.T, suffix string) {
255 t.Helper()
256 checker := &coordChecker{}
257 backend := &backendStub{}
258 s := startServer(t, checker, backend)
260 res := get(t, s, suffix)
262 assert.Equal(t, http.StatusNotFound, res.status)
263 assert.Zero(t, backend.count(), "a refused path must never be delegated")
264 assert.Empty(t, checker.coordinates())
267func TestGateRefusesDotSegments(t *testing.T) {
268 t.Parallel()
269 assertRefused(t, "/github.com/zzz/../foo/bar/@v/v1.0.0.zip")
270 assertRefused(t, "/sumdb/sum.golang.org/../../github.com/pkg/errors/@v/v0.9.1.zip")
273func TestGateRefusesTrailingSlash(t *testing.T) {
274 t.Parallel()
275 assertRefused(t, "/github.com/pkg/errors/@v/v0.9.1.zip/")
278func TestGateRefusesEmptyModulePath(t *testing.T) {
279 t.Parallel()
280 assertRefused(t, "/@v/v1.0.0.zip")
283func TestGateRefusesSecondAtV(t *testing.T) {
284 t.Parallel()
285 t.Run("empty first module", func(t *testing.T) {
286 t.Parallel()
287 assertRefused(t, "/@v/x/github.com/foo/bar/@v/v1.0.0.zip")
288 })
289 t.Run("second segment lands in the version", func(t *testing.T) {
290 t.Parallel()
291 assertRefused(t, "/github.com/foo/@v/x/github.com/foo/bar/@v/v1.0.0.zip")
292 })
295func TestGateRefusesMalformedBangEscapes(t *testing.T) {
296 t.Parallel()
297 for _, elem := range []string{"!!a", "a!", "!A", "Foo"} {
298 t.Run(elem, func(t *testing.T) {
299 t.Parallel()
300 assertRefused(t, "/github.com/"+elem+"/m/@v/v1.0.0.zip")
301 })
302 }
305func TestHeadRequestIsAllowed(t *testing.T) {
306 t.Parallel()
307 checker := &coordChecker{}
308 backend := &backendStub{}
309 s := startServer(t, checker, backend)
311 res := do(t, http.MethodHead, s.BaseURL()+"/github.com/pkg/errors/@v/v0.9.1.info")
312 assert.NotEqual(t, http.StatusMethodNotAllowed, res.status)
313 assert.Equal(t, 1, backend.count())
316// disconnectChecker blocks until released and reports whether its context
317// was cancelled first.
318type disconnectChecker struct {
319 entered chan struct{}
320 release chan struct{}
321 cancelled chan struct{}
324func (c *disconnectChecker) Check(ctx context.Context, _ policy.Request) (policy.Result, error) {
325 close(c.entered)
326 select {
327 case <-ctx.Done():
328 close(c.cancelled)
329 return policy.Result{}, ctx.Err()
330 case <-c.release:
331 return policy.Result{}, nil
332 }
335func TestClientDisconnectDoesNotRecordBlocked(t *testing.T) {
336 t.Parallel()
337 checker := &disconnectChecker{
338 entered: make(chan struct{}),
339 release: make(chan struct{}),
340 cancelled: make(chan struct{}),
341 }
342 s := startServer(t, checker, &backendStub{})
344 ctx, cancel := context.WithCancel(t.Context())
345 req, err := http.NewRequestWithContext(ctx, http.MethodGet,
346 s.BaseURL()+"/github.com/pkg/errors/@v/v0.9.1.zip", http.NoBody)
347 require.NoError(t, err)
348 done := make(chan struct{})
349 go func() {
350 defer close(done)
351 res, err := http.DefaultClient.Do(req)
352 if err == nil {
353 _ = res.Body.Close()
354 }
355 }()
357 <-checker.entered
358 cancel()
359 <-done
360 // Give the server time to notice the disconnect before releasing.
361 select {
362 case <-checker.cancelled:
363 case <-time.After(200 * time.Millisecond):
364 }
365 close(checker.release)
366 s.Stop()
368 select {
369 case <-checker.cancelled:
370 assert.Fail(t, "a client disconnect cancelled the policy check")
371 default:
372 }
373 assert.Empty(t, s.Verdicts(), "a client disconnect must not be recorded as a firewall block")
Diagnostics7 git calls · 37 ms
CallStatusSizeTime037 ms
discover200185 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit5 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=5
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=5
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:0200379 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=379
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=21702eb6df99111b4d7a039e655c720d4209d5b6
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=379
git.pkt.payload.max=352
git.sideband.band1.bytes=352
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=352
git.pack.trailer=7d54607262a7ed73d2686c4df38d0272960c21fc
Open pack map
fetch2002.9 KiB · hit24 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=24
git.response.bytes=2945
git.response.cache=HIT
git.response.duration_ms=24
git.fetch.wants=416eb8e926ec90f8787fd8206b8d229dd213d202
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2945
git.pkt.payload.max=2918
git.sideband.band1.bytes=2918
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e274649e8dd806
Request
0cb97d98…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format