GitLabgitlab-org

cli

main
ply7 git calls216 ms99 KiB4 cachedprotocol v2
356 lines14 KiBRaw
1//go:build !integration
3package gomodproxy
5import (
6 "bytes"
7 "io"
8 "log"
9 "log/slog"
10 "net"
11 "net/http"
12 "net/http/httptest"
13 "os"
14 "os/exec"
15 "path/filepath"
16 "strings"
17 "testing"
19 "github.com/goproxy/goproxy"
20 "github.com/stretchr/testify/assert"
21 "github.com/stretchr/testify/require"
22 "golang.org/x/mod/module"
23 modzip "golang.org/x/mod/zip"
26func envValue(env []string, key string) (string, bool) {
27 value, found := "", false
28 for _, entry := range env {
29 if k, v, ok := strings.Cut(entry, "="); ok && k == key {
30 value, found = v, true
31 }
32 }
33 return value, found
36func TestFetcherEnvInheritsProcessEnvironment(t *testing.T) {
37 t.Setenv("HOME", "/tmp/home-fixture")
38 t.Setenv("GOCACHE", "/tmp/gocache-fixture")
40 env := fetcherEnv(t.TempDir(), nil)
41 for _, key := range []string{"PATH", "HOME", "GOCACHE"} {
42 _, ok := envValue(env, key)
43 assert.True(t, ok, "%s must be inherited", key)
44 }
47func TestFetcherEnvPinsFetcherSettings(t *testing.T) {
48 t.Setenv("GOTOOLCHAIN", "go1.99.0")
49 t.Setenv("GOWORK", "/elsewhere/go.work")
50 t.Setenv("GOFLAGS", "-mod=vendor")
51 t.Setenv("GOMODCACHE", "/elsewhere/modcache")
52 t.Setenv("GO111MODULE", "off")
53 tempDir := t.TempDir()
55 env := fetcherEnv(tempDir, nil)
57 toolchain, _ := envValue(env, "GOTOOLCHAIN")
58 assert.Equal(t, "local", toolchain)
59 work, _ := envValue(env, "GOWORK")
60 assert.Equal(t, "off", work)
61 flags, _ := envValue(env, "GOFLAGS")
62 assert.Equal(t, "-mod=mod -modcacherw", flags)
63 modcache, _ := envValue(env, "GOMODCACHE")
64 assert.Equal(t, filepath.Join(tempDir, "modcache"), modcache)
65 module, _ := envValue(env, "GO111MODULE")
66 assert.Equal(t, "on", module)
69func TestFetcherEnvOverrideReplacesInheritedValue(t *testing.T) {
70 t.Setenv("GOSUMDB", "off")
71 t.Setenv("GOPROXY", "https://inherited.example")
73 env := fetcherEnv(t.TempDir(), map[string]string{
74 "GOSUMDB": "sum.golang.org",
75 "GOPROXY": "corp.example|https://user:pw@mirror.example,direct",
76 })
78 sumdb, _ := envValue(env, "GOSUMDB")
79 assert.Equal(t, "sum.golang.org", sumdb)
80 proxy, _ := envValue(env, "GOPROXY")
81 assert.Equal(t, "corp.example|https://user:pw@mirror.example,direct", proxy)
84func TestFetcherEnvOverrideCannotReplaceIsolationPins(t *testing.T) {
85 t.Parallel()
86 tempDir := t.TempDir()
88 env := fetcherEnv(tempDir, map[string]string{
89 "GOMODCACHE": "/elsewhere/modcache",
90 "GOFLAGS": "-mod=vendor",
91 "GOTOOLCHAIN": "go1.99.0",
92 "GOWORK": "/elsewhere/go.work",
93 })
95 modcache, _ := envValue(env, "GOMODCACHE")
96 assert.Equal(t, filepath.Join(tempDir, "modcache"), modcache)
97 flags, _ := envValue(env, "GOFLAGS")
98 assert.Equal(t, "-mod=mod -modcacherw", flags)
99 toolchain, _ := envValue(env, "GOTOOLCHAIN")
100 assert.Equal(t, "local", toolchain)
101 work, _ := envValue(env, "GOWORK")
102 assert.Equal(t, "off", work)
105func fetcherOf(t *testing.T, b *Backend) *goFetcher {
106 t.Helper()
107 gp, ok := b.handler.(*goproxy.Goproxy)
108 require.True(t, ok)
109 fetcher, ok := gp.Fetcher.(*goFetcher)
110 require.True(t, ok)
111 return fetcher
114func TestNewBackendUpstreamCAs(t *testing.T) {
115 caPEM := []byte("-----BEGIN CERTIFICATE-----\nfixture\n-----END CERTIFICATE-----\n")
116 cases := []struct {
117 name string
118 goos string
119 certFile string
120 certDir string
121 upstreamCAs []byte
122 wantOwnFile bool
123 }{
124 {name: "no extra CAs", goos: "linux", certFile: "/user/bundle.pem"},
125 {name: "extra CAs", goos: "linux", upstreamCAs: caPEM, wantOwnFile: true},
126 {name: "extra CAs over the user's SSL_CERT_FILE", goos: "linux", certFile: "/user/bundle.pem", upstreamCAs: caPEM, wantOwnFile: true},
127 {name: "extra CAs with SSL_CERT_FILE and SSL_CERT_DIR", goos: "linux", certFile: "/user/bundle.pem", certDir: "/user/certs", upstreamCAs: caPEM, wantOwnFile: true},
128 {name: "extra CAs with only SSL_CERT_DIR", goos: "linux", certDir: "/user/certs", upstreamCAs: caPEM},
129 {name: "extra CAs on macOS", goos: "darwin", certFile: "/user/bundle.pem", upstreamCAs: caPEM},
130 {name: "extra CAs on Windows", goos: "windows", upstreamCAs: caPEM},
131 }
132 for _, c := range cases {
133 t.Run(c.name, func(t *testing.T) {
134 t.Setenv("SSL_CERT_FILE", c.certFile)
135 t.Setenv("SSL_CERT_DIR", c.certDir)
136 b, err := newBackend(c.goos, "go", nil, map[string]string{"GOSUMDB": "off"}, c.upstreamCAs)
137 require.NoError(t, err)
138 t.Cleanup(b.Close)
140 certFile, _ := envValue(fetcherOf(t, b).env, "SSL_CERT_FILE")
141 ownFile := filepath.Join(b.tempDir, "upstream-ca.pem")
142 if c.wantOwnFile {
143 assert.Equal(t, ownFile, certFile)
144 got, err := os.ReadFile(ownFile)
145 require.NoError(t, err)
146 assert.Equal(t, caPEM, got)
147 } else {
148 assert.Equal(t, c.certFile, certFile)
149 assert.NoFileExists(t, ownFile)
150 }
151 gp, ok := b.handler.(*goproxy.Goproxy)
152 require.True(t, ok)
153 assert.Equal(t, c.upstreamCAs != nil, gp.Transport != nil)
154 })
155 }
158func TestNewBackendConfiguresSumDBAndLogger(t *testing.T) {
159 t.Parallel()
160 b, err := NewBackend("/usr/bin/go", []string{"sum.golang.org"}, map[string]string{"GOSUMDB": "sum.golang.org"}, nil)
161 require.NoError(t, err)
162 t.Cleanup(b.Close)
164 gp, ok := b.handler.(*goproxy.Goproxy)
165 require.True(t, ok)
166 assert.Equal(t, []string{"sum.golang.org"}, gp.ProxiedSumDBs)
167 assert.NotNil(t, gp.Cacher, "a nil Cacher refetches upstream on every request")
168 require.NotNil(t, gp.Logger)
169 assert.IsType(t, debugHandler{}, gp.Logger.Handler())
171 fetcher := fetcherOf(t, b)
172 assert.Equal(t, "/usr/bin/go", fetcher.goBin)
173 sumdb, _ := envValue(fetcher.env, "GOSUMDB")
174 assert.Equal(t, "sum.golang.org", sumdb)
175 entries, err := os.ReadDir(fetcher.dir)
176 require.NoError(t, err)
177 assert.Empty(t, entries, "go must run where no go.mod applies")
180func writeFileProxy(t *testing.T) string {
181 t.Helper()
182 src := t.TempDir()
183 require.NoError(t, os.WriteFile(filepath.Join(src, "go.mod"), []byte("module example.com/m\n"), 0o600))
184 require.NoError(t, os.WriteFile(filepath.Join(src, "m.go"), []byte("package m\n"), 0o600))
186 root := t.TempDir()
187 dir := filepath.Join(root, "example.com", "m", "@v")
188 require.NoError(t, os.MkdirAll(dir, 0o755))
189 require.NoError(t, os.WriteFile(filepath.Join(dir, "list"), []byte("v1.0.0\n"), 0o600))
190 require.NoError(t, os.WriteFile(filepath.Join(dir, "v1.0.0.info"), []byte(`{"Version":"v1.0.0"}`), 0o600))
191 require.NoError(t, os.WriteFile(filepath.Join(dir, "v1.0.0.mod"), []byte("module example.com/m\n"), 0o600))
193 zf, err := os.Create(filepath.Join(dir, "v1.0.0.zip"))
194 require.NoError(t, err)
195 require.NoError(t, modzip.CreateFromDir(zf, module.Version{Path: "example.com/m", Version: "v1.0.0"}, src))
196 require.NoError(t, zf.Close())
197 return root
200func TestBackendCloseRemovesPopulatedModuleCache(t *testing.T) {
201 t.Parallel()
202 b, err := NewBackend("go", nil, map[string]string{
203 "GOPROXY": "file://" + filepath.ToSlash(writeFileProxy(t)),
204 "GOSUMDB": "off",
205 }, nil)
206 require.NoError(t, err)
208 cmd := exec.Command("go", "mod", "download", "-json", "example.com/m@v1.0.0")
209 cmd.Env = fetcherOf(t, b).env
210 cmd.Dir = t.TempDir()
211 out, err := cmd.CombinedOutput()
212 require.NoError(t, err, string(out))
213 _, err = os.Stat(filepath.Join(b.tempDir, "modcache", "example.com", "m@v1.0.0", "m.go"))
214 require.NoError(t, err, "go mod download must populate the private module cache")
216 b.Close()
217 _, err = os.Stat(b.tempDir)
218 assert.True(t, os.IsNotExist(err), "temp tree must not survive Close")
221func serveBackend(t *testing.T, goproxy, target string) *httptest.ResponseRecorder {
222 t.Helper()
223 if _, err := exec.LookPath("go"); err != nil {
224 t.Skip("go not on PATH")
225 }
226 b, err := NewBackend("go", nil, map[string]string{"GOPROXY": goproxy, "GOSUMDB": "off"}, nil)
227 require.NoError(t, err)
228 t.Cleanup(b.Close)
229 rec := httptest.NewRecorder()
230 b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, target, http.NoBody))
231 return rec
234func TestBackendFetchesEachRequestKindThroughGo(t *testing.T) {
235 t.Parallel()
236 goproxy := "file://" + filepath.ToSlash(writeFileProxy(t))
237 for target, want := range map[string]string{
238 "/example.com/m/@v/list": "v1.0.0",
239 "/example.com/m/@latest": `"Version":"v1.0.0"`,
240 "/example.com/m/@v/v1.0.0.mod": "module example.com/m",
241 "/example.com/m/@v/v1.0.0.info": `"Version":"v1.0.0"`,
242 } {
243 t.Run(target, func(t *testing.T) {
244 t.Parallel()
245 rec := serveBackend(t, goproxy, target)
246 assert.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
247 assert.Contains(t, rec.Body.String(), want)
248 })
249 }
252func TestBackendAnswersMissingModuleWith404(t *testing.T) {
253 t.Parallel()
254 rec := serveBackend(t, "file://"+filepath.ToSlash(writeFileProxy(t)), "/example.com/missing/@v/v1.0.0.info")
255 assert.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String())
258var _ io.ReaderFrom = (*failureWriter)(nil)
260func TestBackendFetchesWithModulesDisabledInEnvironment(t *testing.T) {
261 t.Setenv("GO111MODULE", "off")
262 rec := serveBackend(t, "file://"+filepath.ToSlash(writeFileProxy(t)), "/example.com/m/@v/v1.0.0.info")
263 assert.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
266func TestBackendListsRetractedVersions(t *testing.T) {
267 t.Parallel()
268 root := writeFileProxy(t)
269 dir := filepath.Join(root, "example.com", "m", "@v")
270 require.NoError(t, os.WriteFile(filepath.Join(dir, "list"), []byte("v1.0.0\nv1.1.0\n"), 0o600))
271 require.NoError(t, os.WriteFile(filepath.Join(dir, "v1.1.0.info"), []byte(`{"Version":"v1.1.0"}`), 0o600))
272 require.NoError(t, os.WriteFile(filepath.Join(dir, "v1.1.0.mod"), []byte("module example.com/m\n\nretract v1.0.0\n"), 0o600))
274 rec := serveBackend(t, "file://"+filepath.ToSlash(root), "/example.com/m/@v/list")
275 require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
276 assert.Equal(t, []string{"v1.0.0", "v1.1.0"}, strings.Fields(rec.Body.String()), "the child applies retractions itself")
279func TestDebugHandlerRedactsURLUserinfo(t *testing.T) {
280 t.Setenv("GLAB_DEBUG", "true")
281 var out bytes.Buffer
282 log.SetOutput(&out)
283 t.Cleanup(func() { log.SetOutput(os.Stderr) })
285 slog.New(debugHandler{}).Error("failed to proxy checksum database",
286 "error", `Get "https://t0ken@sum.corp.example/latest": dial tcp: connection refused`)
287 assert.Contains(t, out.String(), "https://xxxxx@sum.corp.example/latest")
288 assert.NotContains(t, out.String(), "t0ken")
291func TestBackendServesCachedZipIntact(t *testing.T) {
292 t.Parallel()
293 if _, err := exec.LookPath("go"); err != nil {
294 t.Skip("go not on PATH")
295 }
296 root := writeFileProxy(t)
297 want, err := os.ReadFile(filepath.Join(root, "example.com", "m", "@v", "v1.0.0.zip"))
298 require.NoError(t, err)
299 b, err := NewBackend("go", nil, map[string]string{"GOPROXY": "file://" + filepath.ToSlash(root), "GOSUMDB": "off"}, nil)
300 require.NoError(t, err)
301 t.Cleanup(b.Close)
302 srv := httptest.NewServer(b)
303 t.Cleanup(srv.Close)
305 for _, pass := range []string{"fetched", "cached"} {
306 resp, err := http.Get(srv.URL + "/example.com/m/@v/v1.0.0.zip")
307 require.NoError(t, err, pass)
308 got, err := io.ReadAll(resp.Body)
309 require.NoError(t, resp.Body.Close())
310 require.NoError(t, err, pass)
311 assert.Equal(t, http.StatusOK, resp.StatusCode, pass)
312 assert.Equal(t, want, got, pass)
313 }
316func TestBackendAnswersUpstreamFailureWithItsMessage(t *testing.T) {
317 t.Parallel()
318 l, err := net.Listen("tcp", "127.0.0.1:0")
319 require.NoError(t, err)
320 addr := l.Addr().String()
321 require.NoError(t, l.Close())
323 rec := serveBackend(t, "https://t0ken@"+addr, "/example.com/m/@v/v1.0.0.info")
324 assert.Equal(t, http.StatusBadGateway, rec.Code)
325 assert.Contains(t, rec.Body.String(), "connection refused")
326 assert.NotContains(t, rec.Body.String(), "t0ken")
329func TestIsNotFound(t *testing.T) {
330 t.Parallel()
331 // goproxy's GoFetcher treated every go failure as not-found; a 404 keeps
332 // go probing shorter module paths for private repos and still carries
333 // go's message.
334 for msg, want := range map[string]bool{
335 "corp.example/m@v1.0.0: invalid version: git ls-remote -q origin in /tmp/x: exit status 128:\n\tgit@corp.example: Permission denied (publickey).\n\tfatal: Could not read from remote repository.": true,
336 "unrecognized import path \"corp.invalid/m\": https fetch: Get \"https://corp.invalid/m?go-get=1\": dial tcp: lookup corp.invalid: no such host": true,
337 "example.com/m@v1.0.0: reading https://corp.example/example.com/m/@v/v1.0.0.info: 404 Not Found": true,
338 "example.com/m@v1.0.0: reading https://corp.example/example.com/m/@v/v1.0.0.info: 410 Gone": true,
339 "example.com/m@v9.9.9: invalid version: unknown revision v9.9.9": true,
340 "module example.com/m: no matching versions for query \"v2\"": true,
341 "unrecognized import path \"corp.example/m\": https fetch: 404": true,
342 "module lookup disabled by GOPROXY=off": true,
343 "example.com/m@v1.0.0: reading file:///srv/proxy/example.com/m/@v/v1.0.0.info: no such file or directory": true,
344 "example.com/m@v1.0.0: reading https://corp.example/example.com/m/@v/v1.0.0.info: 401 Unauthorized": false,
345 "Get \"https://corp.example/example.com/m/@v/list\": dial tcp: connection refused": false,
346 } {
347 assert.Equal(t, want, isNotFound(msg), msg)
348 }
351func TestRedactUserinfo(t *testing.T) {
352 t.Parallel()
353 assert.Equal(t,
354 `Get "https://xxxxx@corp.example/m/@v/list" and http://xxxxx@mirror.example/x`,
355 redactUserinfo(`Get "https://t0ken@corp.example/m/@v/list" and http://alice:xxxxx@mirror.example/x`))
Diagnostics7 git calls · 216 ms
CallStatusSizeTime0216 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchtree:0200361 B · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:0200379 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=379
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=21702eb6df99111b4d7a039e655c720d4209d5b6
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=379
git.pkt.payload.max=352
git.sideband.band1.bytes=352
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=352
git.pack.trailer=7d54607262a7ed73d2686c4df38d0272960c21fc
Open pack map
fetch2003.7 KiB206 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=206
git.response.bytes=3827
git.response.cache=MISS
git.response.duration_ms=206
git.fetch.wants=acd61202897db704526134f51589967bd2c621f2
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3827
git.pkt.payload.max=3800
git.sideband.band1.bytes=3800
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e264c34a5cdf63
Request
9d1fd667…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format