GitLabgitlab-org
cli
main
1//go:build !integration
3package pm
5import (
6 "context"
7 "encoding/json"
8 "errors"
9 "io"
10 "net/http"
11 "os"
12 "path/filepath"
13 "strings"
14 "testing"
16 "github.com/stretchr/testify/assert"
17 "github.com/stretchr/testify/require"
19 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
20 "gitlab.com/gitlab-org/cli/internal/testing/cmdtest"
21)
23type fakeGoExecutor struct {
24 settings map[string]string
25 envErr error
27 calls [][]string
28 preflightEnv []string
29 childEnv []string
30}
32func defaultGoSettings() map[string]string {
33 return map[string]string{
34 "GOPROXY": "https://proxy.golang.org,direct",
35 "GONOPROXY": "",
36 "GONOSUMDB": "",
37 "GOPRIVATE": "",
38 "GOSUMDB": "sum.golang.org",
39 "GOINSECURE": "",
40 }
41}
43func (f *fakeGoExecutor) LookPath(file string) (string, error) { return "/bin/" + file, nil }
45func (f *fakeGoExecutor) ExecWithIO(_ context.Context, _ string, args, env []string, _ io.Reader, stdout, stderr io.Writer) error {
46 f.calls = append(f.calls, args)
47 if len(args) > 0 && args[0] == "env" {
48 f.preflightEnv = env
49 if f.envErr != nil {
50 _, _ = io.WriteString(stderr, "go: unknown GOENV file /home/u/go.env\n")
51 return f.envErr
52 }
53 return json.NewEncoder(stdout).Encode(f.settings)
54 }
55 f.childEnv = env
56 return nil
57}
59type allowChecker struct{}
61func (allowChecker) Check(context.Context, policy.Request) (policy.Result, error) {
62 return policy.Result{}, nil
63}
65type fixtureGoBackend struct {
66 built int
67 closed int
68 goBin string
69 proxiedSumDBs []string
70 overrides map[string]string
71 handler http.Handler
72 onClose func()
73}
75func (b *fixtureGoBackend) newBackend(goBin string, proxiedSumDBs []string, overrides map[string]string, _ []byte) (http.Handler, func(), error) {
76 b.built++
77 b.goBin, b.proxiedSumDBs, b.overrides = goBin, proxiedSumDBs, overrides
78 h := b.handler
79 if h == nil {
80 h = http.NotFoundHandler()
81 }
82 return h, func() {
83 b.closed++
84 if b.onClose != nil {
85 b.onClose()
86 }
87 }, nil
88}
90func setupGolang(t *testing.T, m *golangManager, exec Executor) (setupResult, error) {
91 t.Helper()
92 ios, _, _, _ := cmdtest.TestIOStreams()
93 res, err := m.Setup(t.Context(), setupInput{
94 parent: []string{"GOTOOLCHAIN=auto", "HOME=/home/u"},
95 bin: "/opt/go/bin/go",
96 exec: exec,
97 checker: allowChecker{},
98 projectID: "group/proj",
99 io: ios,
100 })
101 if err == nil {
102 t.Cleanup(res.stop)
103 }
104 return res, err
105}
107var _ setupManager = (*golangManager)(nil)
109func TestGolangPassThroughMatcherMatchesNothing(t *testing.T) {
110 t.Parallel()
111 req, err := http.NewRequest(http.MethodGet, "https://proxy.golang.org/github.com/pkg/errors/@v/v0.9.1.zip", http.NoBody)
112 require.NoError(t, err)
113 assert.False(t, Golang().Matcher().Match(req).Matched)
114}
116// TestGolangEnvironmentFailsClosed pins the default the child gets if
117// Setup's GOPROXY never reached it: off, not the inherited GOPROXY.
118func TestGolangEnvironmentFailsClosed(t *testing.T) {
119 t.Parallel()
120 assert.Equal(t, []string{
121 "GOPROXY=off",
122 "GONOPROXY=none",
123 "GO111MODULE=on",
124 "GIT_HTTP_USER_AGENT=",
125 }, Golang().Environment("http://127.0.0.1:9999"))
126}
128func TestGolangRunChildEnvironment(t *testing.T) {
129 t.Setenv("GITLAB_CI", "")
130 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
131 t.Setenv("GIT_HTTP_USER_AGENT", "proxy.golang.org")
132 t.Setenv("GOPROXY", "direct")
133 t.Setenv("GONOPROXY", "*")
134 t.Setenv("GONOSUMDB", "")
135 require.NoError(t, os.Unsetenv("GONOSUMDB"))
136 ios, _, _, _ := cmdtest.TestIOStreams()
137 exec := &fakeGoExecutor{settings: defaultGoSettings()}
138 backend := &fixtureGoBackend{}
139 m := &golangManager{newBackend: backend.newBackend}
141 err := Run(t.Context(), m, RunOptions{
142 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "group/proj", Args: []string{"mod", "download"},
143 })
144 require.NoError(t, err)
145 require.NotNil(t, exec.childEnv, "the child must start")
147 goproxy := lookupEnv(exec.childEnv, "GOPROXY")
148 assert.Equal(t, m.server.BaseURL(), goproxy)
149 assert.True(t, strings.HasPrefix(goproxy, "http://127.0.0.1:"), "the server must be plain HTTP on loopback")
150 assert.NotContains(t, goproxy, ",")
151 assert.NotContains(t, goproxy, "|")
152 assert.Equal(t, "none", lookupEnv(exec.childEnv, "GONOPROXY"))
153 assert.Equal(t, "on", lookupEnv(exec.childEnv, "GO111MODULE"))
154 assert.Contains(t, exec.childEnv, "GIT_HTTP_USER_AGENT=")
155 assert.NotContains(t, exec.childEnv, "GIT_HTTP_USER_AGENT=proxy.golang.org")
156 for _, e := range exec.childEnv {
157 assert.False(t, strings.HasPrefix(e, "GONOSUMDB="), "GONOSUMDB must fall back to the user's value")
158 }
159}
161func TestGolangSetupRunsPreflight(t *testing.T) {
162 t.Parallel()
163 exec := &fakeGoExecutor{settings: defaultGoSettings()}
164 _, err := setupGolang(t, &golangManager{newBackend: (&fixtureGoBackend{}).newBackend}, exec)
165 require.NoError(t, err)
167 require.Len(t, exec.calls, 1)
168 args := exec.calls[0]
169 require.Len(t, args, 10)
170 assert.Equal(t, []string{"env", "-C"}, args[:2])
171 assert.Equal(t, []string{"-json", "GOPROXY", "GONOPROXY", "GONOSUMDB", "GOPRIVATE", "GOSUMDB", "GOINSECURE"}, args[3:])
172 _, statErr := os.Stat(args[2])
173 assert.True(t, os.IsNotExist(statErr), "the preflight directory must be removed")
174 assert.Equal(t, "local", lookupEnv(exec.preflightEnv, "GOTOOLCHAIN"), "the preflight must not download a toolchain")
175 assert.Equal(t, "/home/u", lookupEnv(exec.preflightEnv, "HOME"))
176}
178func TestGolangSetupPassesEffectiveSettingsToFetcher(t *testing.T) {
179 t.Parallel()
180 exec := &fakeGoExecutor{settings: map[string]string{
181 "GOPROXY": "https://proxy.golang.org,direct",
182 "GONOPROXY": "corp.example/*",
183 "GONOSUMDB": "corp.example/*",
184 "GOPRIVATE": "corp.example/*",
185 "GOSUMDB": "sum.golang.org",
186 "GOINSECURE": "",
187 }}
188 backend := &fixtureGoBackend{}
189 m := &golangManager{newBackend: backend.newBackend}
191 res, err := setupGolang(t, m, exec)
192 require.NoError(t, err)
194 assert.Equal(t, "/opt/go/bin/go", backend.goBin)
195 assert.Equal(t, []string{"sum.golang.org"}, backend.proxiedSumDBs)
196 assert.Equal(t, map[string]string{
197 "GOPROXY": "https://proxy.golang.org,direct",
198 "GONOPROXY": "corp.example/*",
199 "GONOSUMDB": "corp.example/*",
200 "GOPRIVATE": "corp.example/*",
201 "GOSUMDB": "sum.golang.org",
202 "GOINSECURE": "none",
203 }, backend.overrides)
204 assert.Equal(t, []string{"GOPROXY=" + m.server.BaseURL()}, res.env)
205}
207func TestGolangChecksumDB(t *testing.T) {
208 t.Parallel()
209 const key = "sumdb.example+1a2b3c4d+AW5vdCBhIHJlYWwga2V5"
210 cases := []struct {
211 gosumdb string
212 child string
213 proxied []string
214 }{
215 {"sum.golang.org", "sum.golang.org", []string{"sum.golang.org"}},
216 {"off", "off", nil},
217 {"", "", nil},
218 {key, key, []string{"sumdb.example"}},
219 {key + " https://mirror.example/sumdb", key, []string{"sumdb.example https://mirror.example/sumdb"}},
220 {"sum.golang.org https://mirror.example", "sum.golang.org", []string{"sum.golang.org https://mirror.example"}},
221 {"sum.golang.google.cn", "sum.golang.org", []string{"sum.golang.org https://sum.golang.google.cn"}},
222 }
223 for _, c := range cases {
224 t.Run(c.gosumdb, func(t *testing.T) {
225 t.Parallel()
226 child, proxied := checksumDB(c.gosumdb)
227 assert.Equal(t, c.child, child)
228 assert.Equal(t, c.proxied, proxied)
229 })
230 }
231}
233func TestGolangRunGOPROXYOffStartsNoServer(t *testing.T) {
234 t.Setenv("GITLAB_CI", "")
235 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
236 ios, _, _, errOut := cmdtest.TestIOStreams()
237 settings := defaultGoSettings()
238 settings["GOPROXY"] = "off"
239 exec := &fakeGoExecutor{settings: settings}
240 backend := &fixtureGoBackend{}
241 m := &golangManager{newBackend: backend.newBackend}
243 err := Run(t.Context(), m, RunOptions{
244 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "p", Args: []string{"mod", "download"},
245 })
246 require.NoError(t, err)
247 assert.Equal(t, "off", lookupEnv(exec.childEnv, "GOPROXY"))
248 assert.Zero(t, backend.built, "GOPROXY=off must start no backend")
249 assert.Nil(t, m.server, "GOPROXY=off must start no server")
250 assert.Zero(t, m.checkedZips())
251 assert.Equal(t, 1, strings.Count(errOut.String(), goNothingCheckedNotice), "a run that checked nothing says so once")
252}
254func TestGolangRunFailsWhenPreflightFails(t *testing.T) {
255 t.Setenv("GITLAB_CI", "")
256 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
257 ios, _, _, _ := cmdtest.TestIOStreams()
258 exec := &fakeGoExecutor{envErr: errors.New("exit status 1")}
259 backend := &fixtureGoBackend{}
261 err := Run(t.Context(), &golangManager{newBackend: backend.newBackend}, RunOptions{
262 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "p", Args: []string{"mod", "download"},
263 })
264 require.ErrorContains(t, err, "exit status 1")
265 require.ErrorContains(t, err, "unknown GOENV file", "go env's stderr is the user's only clue to why the preflight failed")
266 assert.Nil(t, exec.childEnv, "go must not start when the preflight failed")
267 assert.Zero(t, backend.built)
268}
270func TestGolangSetupPassesCustomGOPROXYToFetcherOnly(t *testing.T) {
271 t.Parallel()
272 for _, goproxy := range []string{
273 "https://corp.example",
274 "https://user:secret@corp.example,direct",
275 "corp.example|https://proxy.golang.org|direct",
276 "direct",
277 } {
278 t.Run(goproxy, func(t *testing.T) {
279 t.Parallel()
280 settings := defaultGoSettings()
281 settings["GOPROXY"] = goproxy
282 backend := &fixtureGoBackend{}
283 m := &golangManager{newBackend: backend.newBackend}
285 res, err := setupGolang(t, m, &fakeGoExecutor{settings: settings})
286 require.NoError(t, err)
287 assert.Equal(t, goproxy, backend.overrides["GOPROXY"])
288 assert.Equal(t, []string{"GOPROXY=" + m.server.BaseURL()}, res.env)
289 })
290 }
291}
293func TestGolangSetupFailsWhenBackendFails(t *testing.T) {
294 t.Parallel()
295 m := &golangManager{newBackend: func(string, []string, map[string]string, []byte) (http.Handler, func(), error) {
296 return nil, nil, errors.New("no temp dir")
297 }}
298 _, err := setupGolang(t, m, &fakeGoExecutor{settings: defaultGoSettings()})
299 require.ErrorContains(t, err, "no temp dir")
300 assert.Nil(t, m.server)
301}
303func TestGolangStopStopsServerBeforeClosingBackend(t *testing.T) {
304 t.Parallel()
305 backend := &fixtureGoBackend{}
306 m := &golangManager{newBackend: backend.newBackend}
307 res, err := setupGolang(t, m, &fakeGoExecutor{settings: defaultGoSettings()})
308 require.NoError(t, err)
310 var servingAtClose bool
311 backend.onClose = func() {
312 resp, err := http.Get(m.server.BaseURL() + "/example.com/m/@v/list")
313 if err == nil {
314 _ = resp.Body.Close()
315 servingAtClose = true
316 }
317 }
318 res.stop()
319 assert.Equal(t, 1, backend.closed)
320 assert.False(t, servingAtClose, "the server must stop before the backend closes")
321}
323func TestGolangManagersGetSeparateServersAndTempTrees(t *testing.T) {
324 tmp := t.TempDir()
325 t.Setenv("TMPDIR", tmp)
326 exec := &fakeGoExecutor{settings: defaultGoSettings()}
327 a, ok := Golang().(*golangManager)
328 require.True(t, ok)
329 b, ok := Golang().(*golangManager)
330 require.True(t, ok)
332 resA, err := setupGolang(t, a, exec)
333 require.NoError(t, err)
334 resB, err := setupGolang(t, b, exec)
335 require.NoError(t, err)
337 assert.NotEqual(t, a.server.BaseURL(), b.server.BaseURL())
338 trees, err := filepath.Glob(filepath.Join(tmp, "glab-df-gomodproxy-*"))
339 require.NoError(t, err)
340 assert.Len(t, trees, 2)
342 resA.stop()
343 resB.stop()
344 left, err := os.ReadDir(tmp)
345 require.NoError(t, err)
346 assert.Empty(t, left, "stopping must leave no temp files behind")
347}
Diagnostics7 git calls · 213 ms
CallStatusSizeTime0213 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:02001.3 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=1327 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=2758711b27b3e124482ce287b5df86092bb20d62 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1327 git.pkt.payload.max=1300 git.sideband.band1.bytes=1300 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1300 git.pack.trailer=c83fc6004a9a1360845513c0fb16272b5c14a3e4Open pack map
fetch2003.2 KiB202 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=202 git.response.bytes=3307 git.response.cache=MISS git.response.duration_ms=202 git.fetch.wants=2e1a5a7a68b59aae1c6b6784a8ce4d22d4473d90 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3307 git.pkt.payload.max=3280 git.sideband.band1.bytes=3280 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e1bb5f2f156c75
- Request
- d8a46a5d…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format