GitLabgitlab-org

cli

main
ply7 git calls213 ms100 KiB5 cachedprotocol v2
347 lines11 KiBRaw
1//go:build !integration
3package pm
5import (
6 "context"
7 "encoding/json"
8 "errors"
9 "io"
10 "net/http"
11 "os"
12 "path/filepath"
13 "strings"
14 "testing"
16 "github.com/stretchr/testify/assert"
17 "github.com/stretchr/testify/require"
19 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
20 "gitlab.com/gitlab-org/cli/internal/testing/cmdtest"
23type fakeGoExecutor struct {
24 settings map[string]string
25 envErr error
27 calls [][]string
28 preflightEnv []string
29 childEnv []string
32func defaultGoSettings() map[string]string {
33 return map[string]string{
34 "GOPROXY": "https://proxy.golang.org,direct",
35 "GONOPROXY": "",
36 "GONOSUMDB": "",
37 "GOPRIVATE": "",
38 "GOSUMDB": "sum.golang.org",
39 "GOINSECURE": "",
40 }
43func (f *fakeGoExecutor) LookPath(file string) (string, error) { return "/bin/" + file, nil }
45func (f *fakeGoExecutor) ExecWithIO(_ context.Context, _ string, args, env []string, _ io.Reader, stdout, stderr io.Writer) error {
46 f.calls = append(f.calls, args)
47 if len(args) > 0 && args[0] == "env" {
48 f.preflightEnv = env
49 if f.envErr != nil {
50 _, _ = io.WriteString(stderr, "go: unknown GOENV file /home/u/go.env\n")
51 return f.envErr
52 }
53 return json.NewEncoder(stdout).Encode(f.settings)
54 }
55 f.childEnv = env
56 return nil
59type allowChecker struct{}
61func (allowChecker) Check(context.Context, policy.Request) (policy.Result, error) {
62 return policy.Result{}, nil
65type fixtureGoBackend struct {
66 built int
67 closed int
68 goBin string
69 proxiedSumDBs []string
70 overrides map[string]string
71 handler http.Handler
72 onClose func()
75func (b *fixtureGoBackend) newBackend(goBin string, proxiedSumDBs []string, overrides map[string]string, _ []byte) (http.Handler, func(), error) {
76 b.built++
77 b.goBin, b.proxiedSumDBs, b.overrides = goBin, proxiedSumDBs, overrides
78 h := b.handler
79 if h == nil {
80 h = http.NotFoundHandler()
81 }
82 return h, func() {
83 b.closed++
84 if b.onClose != nil {
85 b.onClose()
86 }
87 }, nil
90func setupGolang(t *testing.T, m *golangManager, exec Executor) (setupResult, error) {
91 t.Helper()
92 ios, _, _, _ := cmdtest.TestIOStreams()
93 res, err := m.Setup(t.Context(), setupInput{
94 parent: []string{"GOTOOLCHAIN=auto", "HOME=/home/u"},
95 bin: "/opt/go/bin/go",
96 exec: exec,
97 checker: allowChecker{},
98 projectID: "group/proj",
99 io: ios,
100 })
101 if err == nil {
102 t.Cleanup(res.stop)
103 }
104 return res, err
107var _ setupManager = (*golangManager)(nil)
109func TestGolangPassThroughMatcherMatchesNothing(t *testing.T) {
110 t.Parallel()
111 req, err := http.NewRequest(http.MethodGet, "https://proxy.golang.org/github.com/pkg/errors/@v/v0.9.1.zip", http.NoBody)
112 require.NoError(t, err)
113 assert.False(t, Golang().Matcher().Match(req).Matched)
116// TestGolangEnvironmentFailsClosed pins the default the child gets if
117// Setup's GOPROXY never reached it: off, not the inherited GOPROXY.
118func TestGolangEnvironmentFailsClosed(t *testing.T) {
119 t.Parallel()
120 assert.Equal(t, []string{
121 "GOPROXY=off",
122 "GONOPROXY=none",
123 "GO111MODULE=on",
124 "GIT_HTTP_USER_AGENT=",
125 }, Golang().Environment("http://127.0.0.1:9999"))
128func TestGolangRunChildEnvironment(t *testing.T) {
129 t.Setenv("GITLAB_CI", "")
130 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
131 t.Setenv("GIT_HTTP_USER_AGENT", "proxy.golang.org")
132 t.Setenv("GOPROXY", "direct")
133 t.Setenv("GONOPROXY", "*")
134 t.Setenv("GONOSUMDB", "")
135 require.NoError(t, os.Unsetenv("GONOSUMDB"))
136 ios, _, _, _ := cmdtest.TestIOStreams()
137 exec := &fakeGoExecutor{settings: defaultGoSettings()}
138 backend := &fixtureGoBackend{}
139 m := &golangManager{newBackend: backend.newBackend}
141 err := Run(t.Context(), m, RunOptions{
142 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "group/proj", Args: []string{"mod", "download"},
143 })
144 require.NoError(t, err)
145 require.NotNil(t, exec.childEnv, "the child must start")
147 goproxy := lookupEnv(exec.childEnv, "GOPROXY")
148 assert.Equal(t, m.server.BaseURL(), goproxy)
149 assert.True(t, strings.HasPrefix(goproxy, "http://127.0.0.1:"), "the server must be plain HTTP on loopback")
150 assert.NotContains(t, goproxy, ",")
151 assert.NotContains(t, goproxy, "|")
152 assert.Equal(t, "none", lookupEnv(exec.childEnv, "GONOPROXY"))
153 assert.Equal(t, "on", lookupEnv(exec.childEnv, "GO111MODULE"))
154 assert.Contains(t, exec.childEnv, "GIT_HTTP_USER_AGENT=")
155 assert.NotContains(t, exec.childEnv, "GIT_HTTP_USER_AGENT=proxy.golang.org")
156 for _, e := range exec.childEnv {
157 assert.False(t, strings.HasPrefix(e, "GONOSUMDB="), "GONOSUMDB must fall back to the user's value")
158 }
161func TestGolangSetupRunsPreflight(t *testing.T) {
162 t.Parallel()
163 exec := &fakeGoExecutor{settings: defaultGoSettings()}
164 _, err := setupGolang(t, &golangManager{newBackend: (&fixtureGoBackend{}).newBackend}, exec)
165 require.NoError(t, err)
167 require.Len(t, exec.calls, 1)
168 args := exec.calls[0]
169 require.Len(t, args, 10)
170 assert.Equal(t, []string{"env", "-C"}, args[:2])
171 assert.Equal(t, []string{"-json", "GOPROXY", "GONOPROXY", "GONOSUMDB", "GOPRIVATE", "GOSUMDB", "GOINSECURE"}, args[3:])
172 _, statErr := os.Stat(args[2])
173 assert.True(t, os.IsNotExist(statErr), "the preflight directory must be removed")
174 assert.Equal(t, "local", lookupEnv(exec.preflightEnv, "GOTOOLCHAIN"), "the preflight must not download a toolchain")
175 assert.Equal(t, "/home/u", lookupEnv(exec.preflightEnv, "HOME"))
178func TestGolangSetupPassesEffectiveSettingsToFetcher(t *testing.T) {
179 t.Parallel()
180 exec := &fakeGoExecutor{settings: map[string]string{
181 "GOPROXY": "https://proxy.golang.org,direct",
182 "GONOPROXY": "corp.example/*",
183 "GONOSUMDB": "corp.example/*",
184 "GOPRIVATE": "corp.example/*",
185 "GOSUMDB": "sum.golang.org",
186 "GOINSECURE": "",
187 }}
188 backend := &fixtureGoBackend{}
189 m := &golangManager{newBackend: backend.newBackend}
191 res, err := setupGolang(t, m, exec)
192 require.NoError(t, err)
194 assert.Equal(t, "/opt/go/bin/go", backend.goBin)
195 assert.Equal(t, []string{"sum.golang.org"}, backend.proxiedSumDBs)
196 assert.Equal(t, map[string]string{
197 "GOPROXY": "https://proxy.golang.org,direct",
198 "GONOPROXY": "corp.example/*",
199 "GONOSUMDB": "corp.example/*",
200 "GOPRIVATE": "corp.example/*",
201 "GOSUMDB": "sum.golang.org",
202 "GOINSECURE": "none",
203 }, backend.overrides)
204 assert.Equal(t, []string{"GOPROXY=" + m.server.BaseURL()}, res.env)
207func TestGolangChecksumDB(t *testing.T) {
208 t.Parallel()
209 const key = "sumdb.example+1a2b3c4d+AW5vdCBhIHJlYWwga2V5"
210 cases := []struct {
211 gosumdb string
212 child string
213 proxied []string
214 }{
215 {"sum.golang.org", "sum.golang.org", []string{"sum.golang.org"}},
216 {"off", "off", nil},
217 {"", "", nil},
218 {key, key, []string{"sumdb.example"}},
219 {key + " https://mirror.example/sumdb", key, []string{"sumdb.example https://mirror.example/sumdb"}},
220 {"sum.golang.org https://mirror.example", "sum.golang.org", []string{"sum.golang.org https://mirror.example"}},
221 {"sum.golang.google.cn", "sum.golang.org", []string{"sum.golang.org https://sum.golang.google.cn"}},
222 }
223 for _, c := range cases {
224 t.Run(c.gosumdb, func(t *testing.T) {
225 t.Parallel()
226 child, proxied := checksumDB(c.gosumdb)
227 assert.Equal(t, c.child, child)
228 assert.Equal(t, c.proxied, proxied)
229 })
230 }
233func TestGolangRunGOPROXYOffStartsNoServer(t *testing.T) {
234 t.Setenv("GITLAB_CI", "")
235 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
236 ios, _, _, errOut := cmdtest.TestIOStreams()
237 settings := defaultGoSettings()
238 settings["GOPROXY"] = "off"
239 exec := &fakeGoExecutor{settings: settings}
240 backend := &fixtureGoBackend{}
241 m := &golangManager{newBackend: backend.newBackend}
243 err := Run(t.Context(), m, RunOptions{
244 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "p", Args: []string{"mod", "download"},
245 })
246 require.NoError(t, err)
247 assert.Equal(t, "off", lookupEnv(exec.childEnv, "GOPROXY"))
248 assert.Zero(t, backend.built, "GOPROXY=off must start no backend")
249 assert.Nil(t, m.server, "GOPROXY=off must start no server")
250 assert.Zero(t, m.checkedZips())
251 assert.Equal(t, 1, strings.Count(errOut.String(), goNothingCheckedNotice), "a run that checked nothing says so once")
254func TestGolangRunFailsWhenPreflightFails(t *testing.T) {
255 t.Setenv("GITLAB_CI", "")
256 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
257 ios, _, _, _ := cmdtest.TestIOStreams()
258 exec := &fakeGoExecutor{envErr: errors.New("exit status 1")}
259 backend := &fixtureGoBackend{}
261 err := Run(t.Context(), &golangManager{newBackend: backend.newBackend}, RunOptions{
262 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "p", Args: []string{"mod", "download"},
263 })
264 require.ErrorContains(t, err, "exit status 1")
265 require.ErrorContains(t, err, "unknown GOENV file", "go env's stderr is the user's only clue to why the preflight failed")
266 assert.Nil(t, exec.childEnv, "go must not start when the preflight failed")
267 assert.Zero(t, backend.built)
270func TestGolangSetupPassesCustomGOPROXYToFetcherOnly(t *testing.T) {
271 t.Parallel()
272 for _, goproxy := range []string{
273 "https://corp.example",
274 "https://user:secret@corp.example,direct",
275 "corp.example|https://proxy.golang.org|direct",
276 "direct",
277 } {
278 t.Run(goproxy, func(t *testing.T) {
279 t.Parallel()
280 settings := defaultGoSettings()
281 settings["GOPROXY"] = goproxy
282 backend := &fixtureGoBackend{}
283 m := &golangManager{newBackend: backend.newBackend}
285 res, err := setupGolang(t, m, &fakeGoExecutor{settings: settings})
286 require.NoError(t, err)
287 assert.Equal(t, goproxy, backend.overrides["GOPROXY"])
288 assert.Equal(t, []string{"GOPROXY=" + m.server.BaseURL()}, res.env)
289 })
290 }
293func TestGolangSetupFailsWhenBackendFails(t *testing.T) {
294 t.Parallel()
295 m := &golangManager{newBackend: func(string, []string, map[string]string, []byte) (http.Handler, func(), error) {
296 return nil, nil, errors.New("no temp dir")
297 }}
298 _, err := setupGolang(t, m, &fakeGoExecutor{settings: defaultGoSettings()})
299 require.ErrorContains(t, err, "no temp dir")
300 assert.Nil(t, m.server)
303func TestGolangStopStopsServerBeforeClosingBackend(t *testing.T) {
304 t.Parallel()
305 backend := &fixtureGoBackend{}
306 m := &golangManager{newBackend: backend.newBackend}
307 res, err := setupGolang(t, m, &fakeGoExecutor{settings: defaultGoSettings()})
308 require.NoError(t, err)
310 var servingAtClose bool
311 backend.onClose = func() {
312 resp, err := http.Get(m.server.BaseURL() + "/example.com/m/@v/list")
313 if err == nil {
314 _ = resp.Body.Close()
315 servingAtClose = true
316 }
317 }
318 res.stop()
319 assert.Equal(t, 1, backend.closed)
320 assert.False(t, servingAtClose, "the server must stop before the backend closes")
323func TestGolangManagersGetSeparateServersAndTempTrees(t *testing.T) {
324 tmp := t.TempDir()
325 t.Setenv("TMPDIR", tmp)
326 exec := &fakeGoExecutor{settings: defaultGoSettings()}
327 a, ok := Golang().(*golangManager)
328 require.True(t, ok)
329 b, ok := Golang().(*golangManager)
330 require.True(t, ok)
332 resA, err := setupGolang(t, a, exec)
333 require.NoError(t, err)
334 resB, err := setupGolang(t, b, exec)
335 require.NoError(t, err)
337 assert.NotEqual(t, a.server.BaseURL(), b.server.BaseURL())
338 trees, err := filepath.Glob(filepath.Join(tmp, "glab-df-gomodproxy-*"))
339 require.NoError(t, err)
340 assert.Len(t, trees, 2)
342 resA.stop()
343 resB.stop()
344 left, err := os.ReadDir(tmp)
345 require.NoError(t, err)
346 assert.Empty(t, left, "stopping must leave no temp files behind")
Diagnostics7 git calls · 213 ms
CallStatusSizeTime0213 ms
discover200185 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.3 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=1327
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=2758711b27b3e124482ce287b5df86092bb20d62
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1327
git.pkt.payload.max=1300
git.sideband.band1.bytes=1300
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1300
git.pack.trailer=c83fc6004a9a1360845513c0fb16272b5c14a3e4
Open pack map
fetch2003.2 KiB202 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=202
git.response.bytes=3307
git.response.cache=MISS
git.response.duration_ms=202
git.fetch.wants=2e1a5a7a68b59aae1c6b6784a8ce4d22d4473d90
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3307
git.pkt.payload.max=3280
git.sideband.band1.bytes=3280
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e1bb5f2f156c75
Request
d8a46a5d…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format