GitLabgitlab-org

cli

main
ply7 git calls246 ms99 KiB4 cachedprotocol v2
415 lines13 KiBRaw
1//go:build !integration
3package policy
5import (
6 "context"
7 "errors"
8 "fmt"
9 "strings"
10 "sync"
11 "sync/atomic"
12 "testing"
13 "time"
15 "github.com/stretchr/testify/assert"
16 "github.com/stretchr/testify/require"
18 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/verdict"
21type countingChecker struct {
22 calls atomic.Int64
23 result Result
24 err error
26 // entered, when non-nil, receives once each time a call enters inner, and
27 // gate, when non-nil, blocks that call until closed. Together they let a
28 // test observe a call in flight and then release it without sleeping, so
29 // the coalescing window is exercised deterministically.
30 entered chan<- struct{}
31 gate <-chan struct{}
34func (c *countingChecker) Check(context.Context, Request) (Result, error) {
35 c.calls.Add(1)
36 if c.entered != nil {
37 c.entered <- struct{}{}
38 }
39 if c.gate != nil {
40 <-c.gate
41 }
42 return c.result, c.err
45// stalledChecker models an API that accepts the request and never answers:
46// it returns only when its context ends.
47type stalledChecker struct{}
49func (stalledChecker) Check(ctx context.Context, _ Request) (Result, error) {
50 <-ctx.Done()
51 return Result{}, ctx.Err()
54type ctxErrChecker struct{}
56func (ctxErrChecker) Check(ctx context.Context, _ Request) (Result, error) {
57 return Result{}, ctx.Err()
60func TestCacheStalledCheckFailsClosedAtCallerDeadline(t *testing.T) {
61 t.Parallel()
62 c := NewCachingChecker(stalledChecker{}, nil)
63 ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
64 defer cancel()
66 type outcome struct {
67 res Result
68 err error
69 }
70 done := make(chan outcome, 1)
71 go func() {
72 res, err := c.Check(ctx, req("npm", "x", "1.0.0"))
73 done <- outcome{res, err}
74 }()
75 select {
76 case out := <-done:
77 require.NoError(t, out.err)
78 assert.Equal(t, verdict.Blocked, out.res.Verdict)
79 assert.Contains(t, out.res.Reason, context.DeadlineExceeded.Error())
80 case <-time.After(5 * time.Second):
81 t.Fatal("Check outlived the caller's deadline, so a stalled policy API hangs the download")
82 }
85func TestCacheCallerCancellationDoesNotEndSharedCheck(t *testing.T) {
86 t.Parallel()
87 ctx, cancel := context.WithCancel(t.Context())
88 cancel()
89 c := NewCachingChecker(ctxErrChecker{}, nil)
91 res, err := c.Check(ctx, req("npm", "x", "1.0.0"))
92 require.NoError(t, err)
93 assert.Equal(t, Result{}, res, "one caller walking away must not fail the check shared with every coalesced caller")
96func TestCacheMemoizesByRequestKey(t *testing.T) {
97 t.Parallel()
98 inner := &countingChecker{result: Result{Verdict: verdict.Warning, Reason: "abc"}}
99 c := NewCachingChecker(inner, nil)
100 r1, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
101 require.NoError(t, err)
102 r2, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
103 require.NoError(t, err)
104 assert.Equal(t, "abc", r1.Reason)
105 assert.Equal(t, "abc", r2.Reason)
106 assert.Equal(t, int64(1), inner.calls.Load(), "second call must hit cache")
109func TestCacheMemoizesBlockedResult(t *testing.T) {
110 t.Parallel()
111 inner := &countingChecker{result: Result{Verdict: verdict.Blocked, Reason: "denied"}}
112 c := NewCachingChecker(inner, nil)
113 r1, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
114 require.NoError(t, err)
115 r2, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
116 require.NoError(t, err)
117 assert.Equal(t, verdict.Blocked, r1.Verdict)
118 assert.Equal(t, verdict.Blocked, r2.Verdict)
119 assert.Equal(t, "denied", r2.Reason)
120 assert.Equal(t, int64(1), inner.calls.Load(), "a blocked result must round-trip through the cache")
123func TestCacheDistinctRequestKeys(t *testing.T) {
124 t.Parallel()
125 inner := &countingChecker{result: Result{}}
126 c := NewCachingChecker(inner, nil)
127 _, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
128 require.NoError(t, err)
129 _, err = c.Check(t.Context(), req("npm", "x", "2.0.0"))
130 require.NoError(t, err)
131 assert.Equal(t, int64(2), inner.calls.Load())
134func TestCacheDistinctByProjectID(t *testing.T) {
135 t.Parallel()
136 inner := &countingChecker{result: Result{}}
137 c := NewCachingChecker(inner, nil)
138 r := req("npm", "x", "1.0.0")
139 rA := r
140 rA.ProjectID = "group/project-a"
141 rB := r
142 rB.ProjectID = "group/project-b"
143 _, err := c.Check(t.Context(), rA)
144 require.NoError(t, err)
145 _, err = c.Check(t.Context(), rB)
146 require.NoError(t, err)
147 assert.Equal(t, int64(2), inner.calls.Load(), "same coordinate in different projects must not share a cache entry")
150func TestCacheDistinctByOperation(t *testing.T) {
151 t.Parallel()
152 inner := &countingChecker{result: Result{}}
153 c := NewCachingChecker(inner, nil)
154 r := req("npm", "x", "1.0.0")
155 rDownload := r
156 rDownload.Operation = Download
157 rUpload := r
158 rUpload.Operation = Upload
159 _, err := c.Check(t.Context(), rDownload)
160 require.NoError(t, err)
161 _, err = c.Check(t.Context(), rUpload)
162 require.NoError(t, err)
163 assert.Equal(t, int64(2), inner.calls.Load(), "same coordinate under different operations must not share a cache entry")
166func TestCacheFailsClosedOnError(t *testing.T) {
167 t.Parallel()
168 inner := &countingChecker{err: errors.New("boom")}
169 c := NewCachingChecker(inner, nil)
170 r, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
171 require.NoError(t, err, "fail-closed returns a blocked Result, not an error")
172 assert.Equal(t, verdict.Blocked, r.Verdict)
173 assert.Contains(t, r.Reason, "policy check failed")
174 assert.True(t, r.CheckFailed, "a block standing in for a failed check must say so")
177func TestCacheCheckFailedOnlyMarksFailures(t *testing.T) {
178 t.Parallel()
179 t.Run("when the policy blocks", func(t *testing.T) {
180 t.Parallel()
181 c := NewCachingChecker(&countingChecker{result: Result{Verdict: verdict.Blocked, Reason: "denied"}}, nil)
182 r, err := c.Check(t.Context(), req("maven", "a:b", "1.0"))
183 require.NoError(t, err)
184 assert.True(t, r.Blocked())
185 assert.False(t, r.CheckFailed, "a rule's block is a decision, not a failure")
186 })
189func TestCacheDoesNotCacheErrors(t *testing.T) {
190 t.Parallel()
191 inner := &countingChecker{err: errors.New("boom")}
192 c := NewCachingChecker(inner, nil)
193 _, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
194 require.NoError(t, err)
195 _, err = c.Check(t.Context(), req("npm", "x", "1.0.0"))
196 require.NoError(t, err)
197 assert.Equal(t, int64(2), inner.calls.Load(), "errored results must not be cached")
200func TestCacheLogsOnError(t *testing.T) {
201 t.Parallel()
202 inner := &countingChecker{err: errors.New("boom")}
203 var gotFormat string
204 var gotArgs []any
205 log := func(format string, a ...any) {
206 gotFormat = format
207 gotArgs = a
208 }
209 c := NewCachingChecker(inner, log)
210 _, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
211 require.NoError(t, err)
212 require.NotEmpty(t, gotFormat, "log callback must be invoked on a failed check")
213 // The key and the underlying error are passed to the callback; the raw
214 // error is not surfaced verbatim in the caller-facing Result.
215 require.Len(t, gotArgs, 2)
216 assert.Contains(t, gotArgs[0], "npm")
217 assert.Contains(t, gotArgs[0], "x")
220func TestCacheFailsOpenWhenFirewallNotEvaluating(t *testing.T) {
221 t.Parallel()
222 inner := &countingChecker{err: errFirewallNotEvaluating}
223 c := NewCachingChecker(inner, nil)
224 r, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
225 require.NoError(t, err)
226 assert.Equal(t, Result{}, r, "not-evaluating maps to allow, not block")
229func TestCacheLogsWhenFirewallNotEvaluating(t *testing.T) {
230 t.Parallel()
231 inner := &countingChecker{err: errFirewallNotEvaluating}
232 var gotFormat string
233 log := func(format string, a ...any) { gotFormat = format }
234 c := NewCachingChecker(inner, log)
235 _, err := c.Check(t.Context(), req("npm", "x", "1.0.0"))
236 require.NoError(t, err)
237 assert.Contains(t, gotFormat, "not evaluating", "an unexpected allow-all must be visible in the log")
240func TestCacheLatchesNotEvaluatingPerProject(t *testing.T) {
241 t.Parallel()
242 // Once a project is observed not evaluating, later coordinates for that
243 // same project short-circuit to allow without another inner call.
244 inner := &countingChecker{err: errFirewallNotEvaluating}
245 c := NewCachingChecker(inner, nil)
247 base := req("npm", "a", "1.0.0")
248 base.ProjectID = "group/project"
250 first := base
251 r1, err := c.Check(t.Context(), first)
252 require.NoError(t, err)
253 assert.Equal(t, Result{}, r1)
255 second := base
256 second.Coordinate = Coordinate{Ecosystem: "npm", Name: "b", Version: "2.0.0"}
257 r2, err := c.Check(t.Context(), second)
258 require.NoError(t, err)
259 assert.Equal(t, Result{}, r2)
261 assert.Equal(t, int64(1), inner.calls.Load(),
262 "a not-evaluating project must be evaluated once, then latched for later coordinates")
265func TestCacheLatchLogsOncePerProject(t *testing.T) {
266 t.Parallel()
267 // The not-evaluating diagnostic is emitted once per project, not once per
268 // coordinate, so a large install does not flood stderr.
269 inner := &countingChecker{err: errFirewallNotEvaluating}
270 var logCount int
271 log := func(string, ...any) { logCount++ }
272 c := NewCachingChecker(inner, log)
274 base := req("npm", "a", "1.0.0")
275 base.ProjectID = "group/project"
277 for i, name := range []string{"a", "b", "c"} {
278 r := base
279 r.Coordinate = Coordinate{Ecosystem: "npm", Name: name, Version: "1.0.0"}
280 res, err := c.Check(t.Context(), r)
281 require.NoError(t, err, "coordinate %d", i)
282 assert.Equal(t, Result{}, res)
283 }
285 assert.Equal(t, 1, logCount, "not-evaluating must be logged once per project, not per coordinate")
288func TestCacheLatchIsPerProject(t *testing.T) {
289 t.Parallel()
290 // Latching one project must not short-circuit a different project: the
291 // second project is still evaluated by inner.
292 inner := &countingChecker{err: errFirewallNotEvaluating}
293 c := NewCachingChecker(inner, nil)
295 rA := req("npm", "x", "1.0.0")
296 rA.ProjectID = "group/project-a"
297 rB := req("npm", "x", "1.0.0")
298 rB.ProjectID = "group/project-b"
300 _, err := c.Check(t.Context(), rA)
301 require.NoError(t, err)
302 _, err = c.Check(t.Context(), rB)
303 require.NoError(t, err)
305 assert.Equal(t, int64(2), inner.calls.Load(),
306 "a not-evaluating latch is per project, so a different project is still evaluated")
309func TestCacheConcurrentSameKeySingleFetch(t *testing.T) {
310 t.Parallel()
311 // The winning caller enters inner and signals on entered; it blocks on
312 // gate while the other callers pile up on the entry's Once. Receiving the
313 // entry signal proves a call is in flight, so we can release it without a
314 // sleep. Coalescing means exactly one call ever enters inner.
315 entered := make(chan struct{})
316 gate := make(chan struct{})
317 inner := &countingChecker{
318 result: Result{Verdict: verdict.Blocked, Reason: "denied"},
319 entered: entered,
320 gate: gate,
321 }
322 c := NewCachingChecker(inner, nil)
323 r := req("npm", "left-pad", "1.3.0")
325 const n = 50
326 var wg sync.WaitGroup
327 results := make([]Result, n)
328 errs := make([]error, n)
329 wg.Add(n)
330 for i := range n {
331 go func() {
332 defer wg.Done()
333 results[i], errs[i] = c.Check(t.Context(), r)
334 }()
335 }
336 // Block until a call is provably in flight, then release it. Any caller
337 // that had to run inner would deadlock here, since the test only ever
338 // drains a single entry signal.
339 <-entered
340 close(gate)
341 wg.Wait()
343 for i := range n {
344 require.NoError(t, errs[i])
345 assert.Equal(t, verdict.Blocked, results[i].Verdict)
346 assert.Equal(t, "denied", results[i].Reason)
347 }
348 // Coalescing collapses concurrent identical requests into one inner call.
349 assert.Equal(t, int64(1), inner.calls.Load(), "concurrent identical requests must coalesce into a single inner check")
352type panickingChecker struct {
353 calls atomic.Int64
354 entered chan<- struct{}
355 gate <-chan struct{}
358func (c *panickingChecker) Check(context.Context, Request) (Result, error) {
359 if c.calls.Add(1) == 1 {
360 c.entered <- struct{}{}
361 <-c.gate
362 panic("boom")
363 }
364 return Result{Verdict: verdict.Blocked, Reason: "denied"}, nil
367func TestCachePanicFailsClosed(t *testing.T) {
368 t.Parallel()
369 entered, gate := make(chan struct{}, 1), make(chan struct{})
370 inner := &panickingChecker{entered: entered, gate: gate}
371 var warned strings.Builder
372 var warnedMu sync.Mutex
373 c := NewCachingChecker(inner, func(format string, a ...any) {
374 warnedMu.Lock()
375 defer warnedMu.Unlock()
376 fmt.Fprintf(&warned, format, a...)
377 })
378 r := req("maven", "a:b", "1.0")
380 type outcome struct {
381 res Result
382 panicked any
383 }
384 first := make(chan outcome, 1)
385 go func() {
386 var o outcome
387 defer func() { o.panicked = recover(); first <- o }()
388 o.res, _ = c.Check(t.Context(), r)
389 }()
390 <-entered
391 c.mu.Lock()
392 waiter := c.cache[r]
393 c.mu.Unlock()
394 close(gate)
395 got := <-first
396 require.Nil(t, got.panicked, "the cache must not pass a checker's panic on to its caller")
397 assert.Equal(t, verdict.Blocked, got.res.Verdict, "the caller that ran the panicked check is blocked")
398 assert.True(t, got.res.CheckFailed)
399 assert.True(t, got.res.Internal, "a glab panic is told apart from a policy service failure")
400 assert.Equal(t, "glab hit an internal error while checking a:b@1.0 and blocked it. Rerun to retry, and report this with the output of GLAB_DEBUG=1.", got.res.Reason)
401 warnedMu.Lock()
402 assert.Equal(t, "warning: glab hit an internal error while checking a:b@1.0. Rerun to retry, and report this with the output of GLAB_DEBUG=1.\n", warned.String(),
403 "the warning names only the checked package: the caller decides the verdict")
404 warnedMu.Unlock()
405 assert.NotContains(t, got.res.Reason, "boom", "the panic value stays out of the reason")
407 waiter.once.Do(func() {})
408 assert.Equal(t, verdict.Blocked, waiter.res.Verdict, "a caller waiting on the panicked check must be blocked")
409 assert.True(t, waiter.res.CheckFailed)
411 again, err := c.Check(t.Context(), r)
412 require.NoError(t, err)
413 assert.Equal(t, int64(2), inner.calls.Load(), "a panicked check must not be cached")
414 assert.Equal(t, "denied", again.Reason)
Diagnostics7 git calls · 246 ms
CallStatusSizeTime0246 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:0200332 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=234
git.response.bytes=332
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=cb813f0d675e9dca14f167e8c7788d61eb19627f
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=332
git.pkt.payload.max=305
git.sideband.band1.bytes=305
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=305
git.pack.trailer=5e210c1831ecf78390a6fe6e1784efd898336da0
Open pack map
fetch2003.5 KiB234 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3624
git.response.cache=MISS
git.response.duration_ms=234
git.fetch.wants=4ae19db8fa22b7391fb93f2fddce27790b432fbf
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3624
git.pkt.payload.max=3597
git.sideband.band1.bytes=3597
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e25af41995a23e
Request
c234dbd7…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format