GitLabgitlab-org

cli

main
ply7 git calls32 ms99 KiB5 cachedprotocol v2
244 lines9.7 KiBRaw
1//go:build !integration
3package proxy
5import (
6 "net/http"
7 "net/http/httptest"
8 "net/url"
9 "testing"
11 "github.com/stretchr/testify/assert"
12 "github.com/stretchr/testify/require"
14 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
17// log4jParentPOM is log4j 2's parent POM behind a mirror. One of its guessed
18// readings is log4j:log4j, the unrelated Log4j 1 package.
19const log4jParentPOM = "/repository/public/org/apache/logging/log4j/log4j/2.24.3/log4j-2.24.3.pom"
21func knownMirrors(t *testing.T, externalOnly bool, raw ...string) *MavenMirrors {
22 t.Helper()
23 urls := make([]*url.URL, 0, len(raw))
24 for _, r := range raw {
25 u, err := url.Parse(r)
26 require.NoError(t, err)
27 urls = append(urls, u)
28 }
29 m := &MavenMirrors{}
30 m.Set(urls, externalOnly)
31 return m
34func TestMavenMirrorKnownURLs(t *testing.T) {
35 t.Parallel()
36 nexus := knownMirrors(t, false, "https://nexus.example.com/repository/public/")
38 t.Run("when a download is under a known mirror", func(t *testing.T) {
39 t.Parallel()
40 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, log4jParentPOM, "https://nexus.example.com"))
42 require.True(t, m.Matched)
43 assert.Equal(t, "org.apache.logging.log4j:log4j", m.Coordinate.Name)
44 assert.Equal(t, "2.24.3", m.Coordinate.Version)
45 assert.Empty(t, m.Alternates, "a known mirror leaves nothing to guess")
46 })
48 t.Run("when a rule names the log4j:log4j reading a guess would make", func(t *testing.T) {
49 t.Parallel()
50 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, log4jParentPOM, "https://nexus.example.com"))
52 p, err := New(MavenMatcher{Mirrors: nexus}, &denyChecker{name: "log4j:log4j"}, "group/proj")
53 require.NoError(t, err)
54 res, _ := p.checkPolicy(t.Context(), m)
55 assert.False(t, res.Blocked(), "log4j 2's parent POM must not be blocked as Log4j 1")
57 p, err = New(MavenMatcher{Mirrors: nexus}, &denyChecker{name: "org.apache.logging.log4j:log4j"}, "group/proj")
58 require.NoError(t, err)
59 res, coord := p.checkPolicy(t.Context(), m)
60 assert.True(t, res.Blocked())
61 assert.Equal(t, "org.apache.logging.log4j:log4j", coord.Name)
62 })
64 t.Run("when two known mirrors nest on one host", func(t *testing.T) {
65 t.Parallel()
66 nested := knownMirrors(t, false, "https://nexus.example.com/repository/", "https://nexus.example.com/repository/central/")
68 m := MavenMatcher{Mirrors: nested}.Match(mvn(http.MethodGet, "/repository/central/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", "https://nexus.example.com"))
70 assert.Equal(t, "org.slf4j:slf4j-api", m.Coordinate.Name, "the longest mirror decides the primary reading")
71 assert.Equal(t, []string{"central.org.slf4j:slf4j-api"}, candidateNames(m)[1:])
72 })
74 t.Run("when the path only shares a prefix with a known mirror", func(t *testing.T) {
75 t.Parallel()
76 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, "/repository/public-snapshots/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", "https://nexus.example.com"))
78 assert.Equal(t, "repository.public-snapshots.org.slf4j:slf4j-api", m.Coordinate.Name, "a mirror path only matches on a segment boundary")
79 assert.NotEmpty(t, m.Alternates)
80 })
82 t.Run("when the mirror URL has no trailing slash", func(t *testing.T) {
83 t.Parallel()
84 bare := knownMirrors(t, false, "https://nexus.example.com/repository/public")
86 under := MavenMatcher{Mirrors: bare}.Match(mvn(http.MethodGet, log4jParentPOM, "https://nexus.example.com"))
87 assert.Empty(t, under.Alternates)
88 beside := MavenMatcher{Mirrors: bare}.Match(mvn(http.MethodGet, "/repository/public-snapshots/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", "https://nexus.example.com"))
89 assert.NotEmpty(t, beside.Alternates, "a mirror path only matches on a segment boundary")
90 })
92 t.Run("when the request names the default port", func(t *testing.T) {
93 t.Parallel()
94 httpMirror := knownMirrors(t, false, "http://nexus.example.com/repository/public/")
95 for _, tc := range []struct {
96 mirrors *MavenMirrors
97 host string
98 }{
99 {nexus, "https://nexus.example.com:443"},
100 {httpMirror, "http://nexus.example.com:80"},
101 {httpMirror, "http://nexus.example.com"},
102 } {
103 m := MavenMatcher{Mirrors: tc.mirrors}.Match(mvn(http.MethodGet, log4jParentPOM, tc.host))
104 assert.Equal(t, "org.apache.logging.log4j:log4j", m.Coordinate.Name, tc.host)
105 assert.Empty(t, m.Alternates, tc.host)
106 }
107 })
109 t.Run("when the request is to another host, port or scheme", func(t *testing.T) {
110 t.Parallel()
111 for _, host := range []string{"https://other.example.com", "https://nexus.example.com:8443", "http://nexus.example.com"} {
112 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, log4jParentPOM, host))
113 assert.NotEmpty(t, m.Alternates, "%s isn't a known mirror, so the prefix is guessed", host)
114 }
115 })
117 t.Run("when the download under a known mirror isn't an artifact", func(t *testing.T) {
118 t.Parallel()
119 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, "/repository/public/org/slf4j/slf4j-api/maven-metadata.xml", "https://nexus.example.com"))
121 assert.False(t, m.Matched)
122 })
124 t.Run("when the filename under a known mirror claims another version", func(t *testing.T) {
125 t.Parallel()
126 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, "/repository/public/org/slf4j/slf4j-api/1/slf4j-api-10.jar", "https://nexus.example.com"))
128 assert.True(t, m.Matched)
129 assert.False(t, m.Pass, "a coordinate the path can't be trusted for fails closed")
130 })
132 t.Run("when the request is an upload under a known mirror", func(t *testing.T) {
133 t.Parallel()
134 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodPut, "/repository/public/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", "https://nexus.example.com"))
136 assert.NotEmpty(t, m.Alternates, "Maven never deploys through a mirror, so an upload's prefix is guessed")
137 })
139 t.Run("when the path under a known mirror has an empty or dot segment", func(t *testing.T) {
140 t.Parallel()
141 for _, path := range []string{
142 "/repository/public/org//slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar",
143 "/repository/public/org/./slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar",
144 "/repository/public/x/../org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar",
145 } {
146 m := MavenMatcher{Mirrors: nexus}.Match(mvn(http.MethodGet, path, "https://nexus.example.com"))
147 assert.NotEmpty(t, m.Alternates, path)
148 }
149 })
151 t.Run("when the catch-all mirror is external:*", func(t *testing.T) {
152 t.Parallel()
153 local := knownMirrors(t, true, "https://nexus.example.com/repository/public/", "http://localhost:8081/repository/local/")
155 remote := MavenMatcher{Mirrors: local}.Match(mvn(http.MethodGet, log4jParentPOM, "https://nexus.example.com"))
156 assert.Empty(t, remote.Alternates)
158 onLocalhost := MavenMatcher{Mirrors: local}.Match(mvn(http.MethodGet, "/repository/local/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar", "http://localhost:8081"))
159 assert.NotEmpty(t, onLocalhost.Alternates, "external:* doesn't cover repositories on localhost")
160 })
162 t.Run("when no mirrors are known", func(t *testing.T) {
163 t.Parallel()
164 for _, mm := range []MavenMatcher{{}, {Mirrors: &MavenMirrors{}}} {
165 m := mm.Match(mvn(http.MethodGet, log4jParentPOM, "https://nexus.example.com"))
166 assert.Contains(t, candidateNames(m), "log4j:log4j", "without known mirrors the prefix is still guessed")
167 }
168 })
171func TestMavenMirrorKnownURLsThroughTunnel(t *testing.T) {
172 t.Parallel()
174 get := func(t *testing.T, checker policy.Checker) (int, *Proxy) {
175 t.Helper()
176 upstream := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
177 t.Cleanup(upstream.Close)
178 p := startProxyWithMatcher(t, upstream, MavenMatcher{Mirrors: knownMirrors(t, false, upstream.URL+"/repository/public/")}, checker)
179 client := newClient(t, p)
180 defer client.CloseIdleConnections()
181 resp, err := client.Get(upstream.URL + log4jParentPOM)
182 require.NoError(t, err)
183 defer resp.Body.Close()
184 return resp.StatusCode, p
185 }
187 t.Run("when a rule names Log4j 1", func(t *testing.T) {
188 t.Parallel()
189 status, p := get(t, policy.NewCachingChecker(&denyChecker{name: "log4j:log4j"}, nil))
191 assert.Equal(t, http.StatusOK, status)
192 assert.Empty(t, p.Verdicts())
193 })
195 t.Run("when a rule names the real parent POM", func(t *testing.T) {
196 t.Parallel()
197 status, p := get(t, policy.NewCachingChecker(&denyChecker{name: "org.apache.logging.log4j:log4j"}, nil))
199 assert.Equal(t, http.StatusForbidden, status)
200 verdicts := p.Verdicts()
201 require.Len(t, verdicts, 1)
202 assert.Equal(t, "org.apache.logging.log4j:log4j", verdicts[0].Package)
203 })
206func TestMavenMirrorKnownURLsOverPlainHTTP(t *testing.T) {
207 t.Parallel()
209 get := func(t *testing.T, checker policy.Checker) (int, *Proxy) {
210 t.Helper()
211 upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
212 t.Cleanup(upstream.Close)
213 p, err := New(MavenMatcher{Mirrors: knownMirrors(t, false, upstream.URL+"/repository/public/")}, checker, "group/proj")
214 require.NoError(t, err)
215 require.NoError(t, p.Start())
216 t.Cleanup(p.Stop)
217 proxyURL, err := url.Parse("http://" + p.Addr())
218 require.NoError(t, err)
219 client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
220 defer client.CloseIdleConnections()
221 resp, err := client.Get(upstream.URL + log4jParentPOM)
222 require.NoError(t, err)
223 defer resp.Body.Close()
224 return resp.StatusCode, p
225 }
227 t.Run("when a rule names Log4j 1", func(t *testing.T) {
228 t.Parallel()
229 status, p := get(t, policy.NewCachingChecker(&denyChecker{name: "log4j:log4j"}, nil))
231 assert.Equal(t, http.StatusOK, status)
232 assert.Empty(t, p.Verdicts())
233 })
235 t.Run("when a rule names the real parent POM", func(t *testing.T) {
236 t.Parallel()
237 status, p := get(t, policy.NewCachingChecker(&denyChecker{name: "org.apache.logging.log4j:log4j"}, nil))
239 assert.Equal(t, http.StatusForbidden, status)
240 verdicts := p.Verdicts()
241 require.Len(t, verdicts, 1)
242 assert.Equal(t, "org.apache.logging.log4j:log4j", verdicts[0].Package)
243 })
Diagnostics7 git calls · 32 ms
CallStatusSizeTime032 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2002.3 KiB · hit21 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=21
git.response.bytes=2304
git.response.cache=HIT
git.response.duration_ms=21
git.fetch.wants=d806cb03d3cd5fb9881cb066ac1ddb1eb3cc4ba8
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2304
git.pkt.payload.max=2277
git.sideband.band1.bytes=2277
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e2888018e54af4
Request
5605acca…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format