GitLabgitlab-org
cli
main
1//go:build !integration
3package gomodproxy
5import (
6 "context"
7 "io"
8 "net/http"
9 "slices"
10 "strings"
11 "sync"
12 "sync/atomic"
13 "testing"
14 "time"
16 "github.com/goproxy/goproxy"
17 "github.com/stretchr/testify/assert"
18 "github.com/stretchr/testify/require"
20 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
21 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/verdict"
22)
24type coordChecker struct {
25 mu sync.Mutex
26 asked []policy.Request
27 blocked policy.Coordinate
28}
30func (c *coordChecker) Check(_ context.Context, req policy.Request) (policy.Result, error) {
31 c.mu.Lock()
32 c.asked = append(c.asked, req)
33 c.mu.Unlock()
34 if req.Coordinate == c.blocked {
35 return policy.Result{Verdict: verdict.Blocked, Reason: "blocked by policy"}, nil
36 }
37 return policy.Result{}, nil
38}
40func (c *coordChecker) requests() []policy.Request {
41 c.mu.Lock()
42 defer c.mu.Unlock()
43 return slices.Clone(c.asked)
44}
46func (c *coordChecker) coordinates() []policy.Coordinate {
47 var coords []policy.Coordinate
48 for _, r := range c.requests() {
49 coords = append(coords, r.Coordinate)
50 }
51 return coords
52}
54func TestZipCheckSendsFullPolicyRequest(t *testing.T) {
55 t.Parallel()
56 checker := &coordChecker{}
57 s := startServer(t, checker, &backendStub{})
59 res := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
61 assert.Equal(t, http.StatusOK, res.status)
62 assert.Equal(t, []policy.Request{{
63 Coordinate: policy.Coordinate{Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1"},
64 ProjectID: "group/proj",
65 Operation: policy.Download,
66 }}, checker.requests())
67}
69func TestPercentEncodedVersionCannotBypassGate(t *testing.T) {
70 t.Parallel()
71 checker := &coordChecker{blocked: policy.Coordinate{
72 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
73 }}
74 backend := &backendStub{body: "BLOCKED-MODULE-BYTES"}
75 s := startServer(t, checker, backend)
77 plain := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
78 require.Equal(t, http.StatusForbidden, plain.status, "the unencoded form must be blocked")
80 encoded := get(t, s, "/github.com/pkg/errors/@v/v0%2e9%2e1.zip")
81 assert.Equal(t, http.StatusNotFound, encoded.status)
82 assert.NotContains(t, encoded.body, "BLOCKED-MODULE-BYTES")
83 assert.Equal(t, 0, backend.count(), "a percent-encoded path must never reach the backend")
84}
86func TestPercentEncodedPathsAreRefused(t *testing.T) {
87 t.Parallel()
88 checker := &coordChecker{}
89 backend := &backendStub{}
90 s := startServer(t, checker, backend)
92 for _, p := range []string{
93 "/sumdb/%2e%2e/evil.com/@v/v1.0.0.zip",
94 "/%2e%2e/evil.com/@v/v1.0.0.zip",
95 "/evil.com/@v/..%2fv1.0.0.zip",
96 "/evil.com/@v/v1%2e0%2e0.zip",
97 } {
98 res := get(t, s, p)
99 assert.Equal(t, http.StatusNotFound, res.status, p)
100 }
101 assert.Zero(t, backend.count(), "no encoded path may reach the backend")
102 assert.Empty(t, checker.coordinates(), "no encoded path may reach the firewall")
103}
105func TestCaseEncodedModulePathIsGated(t *testing.T) {
106 t.Parallel()
107 checker := &coordChecker{blocked: policy.Coordinate{
108 Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.5.0",
109 }}
110 backend := &backendStub{body: "MODULE-BYTES"}
111 s := startServer(t, checker, backend)
113 info := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.6.0.info")
114 assert.Equal(t, http.StatusOK, info.status)
116 zip := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.6.0.zip")
117 assert.Equal(t, http.StatusOK, zip.status)
118 assert.Equal(t, "MODULE-BYTES", zip.body)
120 blocked := get(t, s, "/github.com/%21burnt%21sushi/toml/@v/v1.5.0.zip")
121 assert.Equal(t, http.StatusForbidden, blocked.status)
122 assert.NotContains(t, blocked.body, "MODULE-BYTES")
124 assert.Equal(t, []policy.Coordinate{
125 {Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.6.0"},
126 {Ecosystem: "golang", Name: "github.com/BurntSushi/toml", Version: "v1.5.0"},
127 }, checker.coordinates())
128 assert.Equal(t, []string{
129 "/github.com/!burnt!sushi/toml/@v/v1.6.0.info",
130 "/github.com/!burnt!sushi/toml/@v/v1.6.0.zip",
131 }, backend.paths, "the backend must read the '!' form the gate checked")
132}
134func TestCaseEncodingDoesNotReopenPercentBypass(t *testing.T) {
135 t.Parallel()
136 checker := &coordChecker{}
137 backend := &backendStub{}
138 s := startServer(t, checker, backend)
140 for _, p := range []string{
141 "/github.com/%21burnt%21sushi/toml/@v/v1%2e6%2e0.zip",
142 "/github.com/%21burnt%21sushi/%2e%2e/evil.com/@v/v1.0.0.zip",
143 "/github.com/%2521burnt/toml/@v/v1.6.0.zip",
144 } {
145 res := get(t, s, p)
146 assert.Equal(t, http.StatusNotFound, res.status, p)
147 }
148 assert.Zero(t, backend.count(), "no other encoding may reach the backend")
149 assert.Empty(t, checker.coordinates(), "no other encoding may reach the firewall")
150}
152type nopSeekCloser struct{ *strings.Reader }
154func (nopSeekCloser) Close() error { return nil }
156type fixtureFetcher struct{ zip string }
158func (f fixtureFetcher) Query(_ context.Context, _, _ string) (string, time.Time, error) {
159 return "v0.9.1", time.Unix(0, 0), nil
160}
162func (f fixtureFetcher) List(_ context.Context, _ string) ([]string, error) {
163 return []string{"v0.9.1"}, nil
164}
166func (f fixtureFetcher) Download(_ context.Context, _, version string) (io.ReadSeekCloser, io.ReadSeekCloser, io.ReadSeekCloser, error) {
167 return nopSeekCloser{strings.NewReader(`{"Version":"` + version + `","Time":"1970-01-01T00:00:00Z"}`)},
168 nopSeekCloser{strings.NewReader("module github.com/pkg/errors\n")},
169 nopSeekCloser{strings.NewReader(f.zip)},
170 nil
171}
173func TestGateAgreesWithRealBackend(t *testing.T) {
174 t.Parallel()
175 checker := &coordChecker{blocked: policy.Coordinate{
176 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
177 }}
178 backend := &goproxy.Goproxy{Fetcher: fixtureFetcher{zip: "REAL-ZIP"}}
179 s := startServer(t, checker, backend)
181 blocked := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
182 assert.Equal(t, http.StatusForbidden, blocked.status)
183 assert.NotContains(t, blocked.body, "REAL-ZIP")
185 encoded := get(t, s, "/github.com/pkg/errors/@v/v0%2e9%2e1.zip")
186 assert.Equal(t, http.StatusNotFound, encoded.status)
187 assert.NotContains(t, encoded.body, "REAL-ZIP")
189 assert.Equal(t, []policy.Coordinate{
190 {Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1"},
191 }, checker.coordinates(), "gate and backend must agree on the coordinate")
192}
194func TestAllowedZipFlowsThroughRealBackend(t *testing.T) {
195 t.Parallel()
196 checker := &coordChecker{}
197 backend := &goproxy.Goproxy{Fetcher: fixtureFetcher{zip: "REAL-ZIP"}}
198 s := startServer(t, checker, backend)
200 res := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
201 assert.Equal(t, http.StatusOK, res.status)
202 assert.Equal(t, "REAL-ZIP", res.body)
204 info := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.info")
205 assert.Equal(t, http.StatusOK, info.status)
206 assert.Contains(t, info.body, "v0.9.1")
207}
209type countingHandler struct {
210 next http.Handler
211 hits atomic.Int64
212}
214func (c *countingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
215 c.hits.Add(1)
216 c.next.ServeHTTP(w, r)
217}
219// Serving the .mod makes goproxy fetch and cache the zip too, so the gate,
220// not the cache, must refuse the zip.
221func TestBlockedZipRefusedAfterBackendCachedModule(t *testing.T) {
222 t.Parallel()
223 checker := &coordChecker{blocked: policy.Coordinate{
224 Ecosystem: "golang", Name: "github.com/pkg/errors", Version: "v0.9.1",
225 }}
226 backend := &countingHandler{next: &goproxy.Goproxy{
227 Fetcher: fixtureFetcher{zip: "REAL-ZIP"},
228 Cacher: goproxy.DirCacher(t.TempDir()),
229 }}
230 s := startServer(t, checker, backend)
232 mod := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.mod")
233 require.Equal(t, http.StatusOK, mod.status)
235 zip := get(t, s, "/github.com/pkg/errors/@v/v0.9.1.zip")
236 assert.Equal(t, http.StatusForbidden, zip.status)
237 assert.NotContains(t, zip.body, "REAL-ZIP")
238 assert.Equal(t, int64(1), backend.hits.Load(), "only the .mod reaches the backend")
239}
241func TestGateChecksUnescapedVersion(t *testing.T) {
242 t.Parallel()
243 checker := &coordChecker{}
244 s := startServer(t, checker, &backendStub{})
246 res := get(t, s, "/example.com/m/@v/v1.0.0-!r!c1.zip")
248 assert.Equal(t, http.StatusOK, res.status)
249 assert.Equal(t, []policy.Coordinate{
250 {Ecosystem: "golang", Name: "example.com/m", Version: "v1.0.0-RC1"},
251 }, checker.coordinates())
252}
254func assertRefused(t *testing.T, suffix string) {
255 t.Helper()
256 checker := &coordChecker{}
257 backend := &backendStub{}
258 s := startServer(t, checker, backend)
260 res := get(t, s, suffix)
262 assert.Equal(t, http.StatusNotFound, res.status)
263 assert.Zero(t, backend.count(), "a refused path must never be delegated")
264 assert.Empty(t, checker.coordinates())
265}
267func TestGateRefusesDotSegments(t *testing.T) {
268 t.Parallel()
269 assertRefused(t, "/github.com/zzz/../foo/bar/@v/v1.0.0.zip")
270 assertRefused(t, "/sumdb/sum.golang.org/../../github.com/pkg/errors/@v/v0.9.1.zip")
271}
273func TestGateRefusesTrailingSlash(t *testing.T) {
274 t.Parallel()
275 assertRefused(t, "/github.com/pkg/errors/@v/v0.9.1.zip/")
276}
278func TestGateRefusesEmptyModulePath(t *testing.T) {
279 t.Parallel()
280 assertRefused(t, "/@v/v1.0.0.zip")
281}
283func TestGateRefusesSecondAtV(t *testing.T) {
284 t.Parallel()
285 t.Run("empty first module", func(t *testing.T) {
286 t.Parallel()
287 assertRefused(t, "/@v/x/github.com/foo/bar/@v/v1.0.0.zip")
288 })
289 t.Run("second segment lands in the version", func(t *testing.T) {
290 t.Parallel()
291 assertRefused(t, "/github.com/foo/@v/x/github.com/foo/bar/@v/v1.0.0.zip")
292 })
293}
295func TestGateRefusesMalformedBangEscapes(t *testing.T) {
296 t.Parallel()
297 for _, elem := range []string{"!!a", "a!", "!A", "Foo"} {
298 t.Run(elem, func(t *testing.T) {
299 t.Parallel()
300 assertRefused(t, "/github.com/"+elem+"/m/@v/v1.0.0.zip")
301 })
302 }
303}
305func TestHeadRequestIsAllowed(t *testing.T) {
306 t.Parallel()
307 checker := &coordChecker{}
308 backend := &backendStub{}
309 s := startServer(t, checker, backend)
311 res := do(t, http.MethodHead, s.BaseURL()+"/github.com/pkg/errors/@v/v0.9.1.info")
312 assert.NotEqual(t, http.StatusMethodNotAllowed, res.status)
313 assert.Equal(t, 1, backend.count())
314}
316// disconnectChecker blocks until released and reports whether its context
317// was cancelled first.
318type disconnectChecker struct {
319 entered chan struct{}
320 release chan struct{}
321 cancelled chan struct{}
322}
324func (c *disconnectChecker) Check(ctx context.Context, _ policy.Request) (policy.Result, error) {
325 close(c.entered)
326 select {
327 case <-ctx.Done():
328 close(c.cancelled)
329 return policy.Result{}, ctx.Err()
330 case <-c.release:
331 return policy.Result{}, nil
332 }
333}
335func TestClientDisconnectDoesNotRecordBlocked(t *testing.T) {
336 t.Parallel()
337 checker := &disconnectChecker{
338 entered: make(chan struct{}),
339 release: make(chan struct{}),
340 cancelled: make(chan struct{}),
341 }
342 s := startServer(t, checker, &backendStub{})
344 ctx, cancel := context.WithCancel(t.Context())
345 req, err := http.NewRequestWithContext(ctx, http.MethodGet,
346 s.BaseURL()+"/github.com/pkg/errors/@v/v0.9.1.zip", http.NoBody)
347 require.NoError(t, err)
348 done := make(chan struct{})
349 go func() {
350 defer close(done)
351 res, err := http.DefaultClient.Do(req)
352 if err == nil {
353 _ = res.Body.Close()
354 }
355 }()
357 <-checker.entered
358 cancel()
359 <-done
360 // Give the server time to notice the disconnect before releasing.
361 select {
362 case <-checker.cancelled:
363 case <-time.After(200 * time.Millisecond):
364 }
365 close(checker.release)
366 s.Stop()
368 select {
369 case <-checker.cancelled:
370 assert.Fail(t, "a client disconnect cancelled the policy check")
371 default:
372 }
373 assert.Empty(t, s.Verdicts(), "a client disconnect must not be recorded as a firewall block")
374}
Diagnostics7 git calls · 37 ms
CallStatusSizeTime037 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit5 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=5 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=5 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:0200379 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=379 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=21702eb6df99111b4d7a039e655c720d4209d5b6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=379 git.pkt.payload.max=352 git.sideband.band1.bytes=352 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=352 git.pack.trailer=7d54607262a7ed73d2686c4df38d0272960c21fcOpen pack map
fetch2002.9 KiB · hit24 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=24 git.response.bytes=2945 git.response.cache=HIT git.response.duration_ms=24 git.fetch.wants=416eb8e926ec90f8787fd8206b8d229dd213d202 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2945 git.pkt.payload.max=2918 git.sideband.band1.bytes=2918 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e274649e8dd806
- Request
- 0cb97d98…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format