GitLabgitlab-org
cli
main
1//go:build !integration
3package pm
5import (
6 "context"
7 "crypto/rand"
8 "encoding/json"
9 "encoding/pem"
10 "io"
11 "log"
12 "net/http"
13 "net/http/httptest"
14 "os"
15 "os/exec"
16 "path/filepath"
17 "runtime"
18 "slices"
19 "strings"
20 "sync/atomic"
21 "testing"
23 "github.com/stretchr/testify/assert"
24 "github.com/stretchr/testify/require"
25 "golang.org/x/mod/module"
26 "golang.org/x/mod/sumdb"
27 "golang.org/x/mod/sumdb/dirhash"
28 "golang.org/x/mod/sumdb/note"
29 modzip "golang.org/x/mod/zip"
31 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/cilog"
32 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/verdict"
33 "gitlab.com/gitlab-org/cli/internal/testing/cmdtest"
34)
36const (
37 fixtureModule = "example.com/m"
38 fixtureVersion = "v1.0.0"
39 fixtureGoMod = "module example.com/m\n"
40)
42type realGoExecutor struct {
43 dir string
44 proxied atomic.Int64
45 proxy string
46 calls [][]string
47 env []string
48}
50func newRealGoExecutor(t *testing.T) *realGoExecutor {
51 t.Helper()
52 e := &realGoExecutor{dir: t.TempDir()}
53 counter := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
54 e.proxied.Add(1)
55 w.WriteHeader(http.StatusBadGateway)
56 }))
57 t.Cleanup(counter.Close)
58 e.proxy = counter.URL
59 return e
60}
62func (e *realGoExecutor) LookPath(file string) (string, error) { return exec.LookPath(file) }
64func (e *realGoExecutor) ExecWithIO(ctx context.Context, name string, args, env []string, stdin io.Reader, stdout, stderr io.Writer) error {
65 e.calls = append(e.calls, args)
66 cmd := exec.CommandContext(ctx, name, args...)
67 cmd.Dir = e.dir
68 cmd.Env = append(slices.Clone(env), "HTTPS_PROXY="+e.proxy, "HTTP_PROXY="+e.proxy)
69 e.env = cmd.Env
70 cmd.Stdin, cmd.Stdout, cmd.Stderr = stdin, stdout, stderr
71 return cmd.Run()
72}
74func isolateGo(t *testing.T, goenv string) {
75 t.Helper()
76 if _, err := exec.LookPath("go"); err != nil {
77 t.Skip("go not on PATH")
78 }
79 envFile := filepath.Join(t.TempDir(), "goenv")
80 require.NoError(t, os.WriteFile(envFile, []byte(goenv), 0o600))
81 t.Setenv("GITLAB_CI", "")
82 t.Setenv("GOENV", envFile)
83 t.Setenv("GOPATH", t.TempDir())
84 t.Setenv("GOMODCACHE", t.TempDir())
85 t.Setenv("GOFLAGS", "-modcacherw")
86 t.Setenv("GOTOOLCHAIN", "local")
87 for _, k := range []string{"GOPROXY", "GONOPROXY", "GONOSUMDB", "GOPRIVATE", "GOSUMDB", "GOINSECURE", "GIT_HTTP_USER_AGENT"} {
88 t.Setenv(k, "")
89 }
90}
92func writeModuleFixture(t *testing.T) (string, []byte) {
93 t.Helper()
94 src := t.TempDir()
95 require.NoError(t, os.WriteFile(filepath.Join(src, "go.mod"), []byte(fixtureGoMod), 0o600))
96 require.NoError(t, os.WriteFile(filepath.Join(src, "m.go"), []byte("package m\n"), 0o600))
98 root := t.TempDir()
99 dir := filepath.Join(root, fixtureModule, "@v")
100 require.NoError(t, os.MkdirAll(dir, 0o755))
101 require.NoError(t, os.WriteFile(filepath.Join(dir, "list"), []byte(fixtureVersion+"\n"), 0o600))
102 require.NoError(t, os.WriteFile(filepath.Join(dir, fixtureVersion+".info"), []byte(`{"Version":"`+fixtureVersion+`"}`), 0o600))
103 require.NoError(t, os.WriteFile(filepath.Join(dir, fixtureVersion+".mod"), []byte(fixtureGoMod), 0o600))
104 zipPath := filepath.Join(dir, fixtureVersion+".zip")
105 zf, err := os.Create(zipPath)
106 require.NoError(t, err)
107 require.NoError(t, modzip.CreateFromDir(zf, module.Version{Path: fixtureModule, Version: fixtureVersion}, src))
108 require.NoError(t, zf.Close())
110 zipHash, err := dirhash.HashZip(zipPath, dirhash.Hash1)
111 require.NoError(t, err)
112 modHash, err := dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
113 return io.NopCloser(strings.NewReader(fixtureGoMod)), nil
114 })
115 require.NoError(t, err)
116 sum := fixtureModule + " " + fixtureVersion + " " + zipHash + "\n" +
117 fixtureModule + " " + fixtureVersion + "/go.mod " + modHash + "\n"
118 return root, []byte(sum)
119}
121type moduleBackend struct {
122 fixtureGoBackend
123 vkey string
124}
126func newModuleBackend(t *testing.T, sumDBName string) *moduleBackend {
127 t.Helper()
128 root, sum := writeModuleFixture(t)
129 mux := http.NewServeMux()
130 mux.Handle("/", http.FileServer(http.Dir(root)))
131 b := &moduleBackend{}
132 if sumDBName != "" {
133 skey, vkey, err := note.GenerateKey(rand.Reader, sumDBName)
134 require.NoError(t, err)
135 b.vkey = vkey
136 db := sumdb.NewServer(sumdb.NewTestServer(skey, func(string, string) ([]byte, error) { return sum, nil }))
137 prefix := "/sumdb/" + sumDBName
138 mux.HandleFunc(prefix+"/supported", func(http.ResponseWriter, *http.Request) {})
139 mux.Handle(prefix+"/", http.StripPrefix(prefix, db))
140 }
141 b.handler = mux
142 return b
143}
145type goRun struct {
146 err error
147 stdout string
148 stderr string
149 dir string
150 exec *realGoExecutor
151 m *golangManager
152 b *moduleBackend
153}
155func runRealGo(t *testing.T, b *moduleBackend, args ...string) goRun {
156 t.Helper()
157 ios, _, out, errOut := cmdtest.TestIOStreams()
158 r := goRun{dir: t.TempDir(), exec: newRealGoExecutor(t), m: &golangManager{newBackend: b.newBackend}, b: b}
159 r.err = Run(t.Context(), r.m, RunOptions{
160 IO: ios, Executor: r.exec, BaseDir: r.dir, ProjectID: "group/proj", Args: args,
161 })
162 r.stdout, r.stderr = out.String(), errOut.String()
163 return r
164}
166func TestRunComposedGolangAllowedModule(t *testing.T) {
167 isolateGo(t, "")
168 t.Setenv("GOSUMDB", "off")
169 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
171 r := runRealGo(t, newModuleBackend(t, ""), "mod", "download", "-json", fixtureModule+"@"+fixtureVersion)
172 require.NoError(t, r.err, r.stderr)
174 got, err := cilog.Load(r.dir)
175 require.NoError(t, err)
176 assert.Empty(t, got.Entries, "an allowed module records no verdict")
177 assert.Equal(t, "go mod download -json "+fixtureModule+"@"+fixtureVersion, got.Session.Command, "the CI log names the command the user ran")
178 assert.Equal(t, 1, r.m.checkedZips())
179 assert.Zero(t, r.exec.proxied.Load(), "no request may reach the MITM proxy")
180 assert.NotContains(t, r.stderr, goNothingCheckedNotice, "a run that checked a zip must not claim it checked nothing")
181}
183// TestRunComposedGolangTrustsConfiguredCAsUpstream proves the go subprocess
184// that fetches modules verifies an HTTPS module proxy against glab's ca_cert
185// and the package-manager CA variables, not only the system roots.
186func TestRunComposedGolangTrustsConfiguredCAsUpstream(t *testing.T) {
187 root, _ := writeModuleFixture(t)
188 var reached atomic.Int64
189 upstream := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
190 reached.Add(1)
191 http.FileServer(http.Dir(root)).ServeHTTP(w, r)
192 }))
193 upstream.Config.ErrorLog = log.New(io.Discard, "", 0)
194 upstream.StartTLS()
195 t.Cleanup(upstream.Close)
196 upstreamCA := filepath.Join(t.TempDir(), "upstream-ca.pem")
197 require.NoError(t, os.WriteFile(upstreamCA, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: upstream.Certificate().Raw}), 0o600))
199 cases := []struct {
200 name string
201 caCertPath string
202 requestsCA string
203 wantOK bool
204 }{
205 {name: "ca_cert", caCertPath: upstreamCA, wantOK: true},
206 {name: "REQUESTS_CA_BUNDLE", requestsCA: upstreamCA, wantOK: true},
207 {name: "neither", wantOK: false},
208 }
209 for _, c := range cases {
210 t.Run(c.name, func(t *testing.T) {
211 if c.wantOK && (runtime.GOOS == "darwin" || runtime.GOOS == "windows") {
212 t.Skip("glab leaves the go subprocess on the platform store on " + runtime.GOOS)
213 }
214 isolateGo(t, "")
215 t.Setenv("GOSUMDB", "off")
216 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
217 for _, v := range upstreamCAVars {
218 t.Setenv(v, "")
219 }
220 t.Setenv("REQUESTS_CA_BUNDLE", c.requestsCA)
221 reached.Store(0)
223 m := &golangManager{newBackend: func(goBin string, proxiedSumDBs []string, overrides map[string]string, upstreamCAs []byte) (http.Handler, func(), error) {
224 overrides["GOPROXY"] = upstream.URL
225 return newGoBackend(goBin, proxiedSumDBs, overrides, upstreamCAs)
226 }}
227 ios, _, _, errOut := cmdtest.TestIOStreams()
228 exec := newRealGoExecutor(t)
229 err := Run(t.Context(), m, RunOptions{
230 IO: ios, Executor: exec, BaseDir: t.TempDir(), ProjectID: "group/proj",
231 Args: []string{"mod", "download", "-json", fixtureModule + "@" + fixtureVersion},
232 CACertPath: c.caCertPath,
233 })
234 if c.wantOK {
235 require.NoError(t, err, errOut.String())
236 assert.Equal(t, 1, m.checkedZips())
237 assert.Positive(t, reached.Load())
238 return
239 }
240 require.Error(t, err)
241 assert.Zero(t, m.checkedZips())
242 assert.Zero(t, reached.Load(), "an untrusted module proxy must not be fetched")
243 })
244 }
245}
247func TestRunComposedGolangNoDownloadPrintsNotice(t *testing.T) {
248 isolateGo(t, "")
249 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
251 r := runRealGo(t, newModuleBackend(t, ""), "env", "GOPROXY")
252 require.NoError(t, r.err, r.stderr)
253 assert.Zero(t, r.m.checkedZips())
254 assert.Equal(t, 1, strings.Count(r.stderr, goNothingCheckedNotice))
255}
257func TestRunComposedGolangBlockedModule(t *testing.T) {
258 cases := map[string]map[string]string{
259 "plain": {},
260 "inherited GOFLAGS": {"GOFLAGS": "-mod=mod -modcacherw"},
261 "inherited GONOPROXY": {"GONOPROXY": "*"},
262 }
263 for name, env := range cases {
264 t.Run(name, func(t *testing.T) {
265 isolateGo(t, "")
266 t.Setenv("GOSUMDB", "off")
267 t.Setenv("GLAB_DF_FAKE_BLOCK", "golang:"+fixtureModule+"@"+fixtureVersion)
268 for k, v := range env {
269 t.Setenv(k, v)
270 }
272 r := runRealGo(t, newModuleBackend(t, ""), "mod", "download", "-json", fixtureModule+"@"+fixtureVersion)
273 var ee *ExitError
274 require.ErrorAs(t, r.err, &ee)
275 assert.True(t, ee.ChildExited, "go itself must fail")
276 assert.Contains(t, r.stdout+r.stderr, "blocked by GLAB_DF_FAKE_BLOCK", "go's output must carry the firewall's reason")
278 got, err := cilog.Load(r.dir)
279 require.NoError(t, err)
280 require.Len(t, got.Entries, 1)
281 assert.Equal(t, fixtureModule, got.Entries[0].Package)
282 assert.Equal(t, verdict.Blocked, got.Entries[0].Verdict)
283 assert.Zero(t, r.exec.proxied.Load(), "no request may reach the MITM proxy")
284 })
285 }
286}
288func TestRunComposedGolangCustomSumDBStaysOnLoopback(t *testing.T) {
289 isolateGo(t, "")
290 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
291 b := newModuleBackend(t, "sumdb.example")
292 t.Setenv("GOSUMDB", b.vkey)
294 r := runRealGo(t, b, "mod", "download", "-json", fixtureModule+"@"+fixtureVersion)
295 require.NoError(t, r.err, r.stderr)
296 assert.Equal(t, []string{"sumdb.example"}, b.proxiedSumDBs)
297 assert.Equal(t, b.vkey, b.overrides["GOSUMDB"])
298 assert.Equal(t, 1, r.m.checkedZips())
299 assert.Zero(t, r.exec.proxied.Load(), "checksum lookups must go to the loopback server")
300}
302func TestRunComposedGolangEnvFileGOPRIVATE(t *testing.T) {
303 isolateGo(t, "GOPRIVATE=corp.example/*\n")
304 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
305 b := newModuleBackend(t, "")
307 r := runRealGo(t, b, "env", "-json", "GOPROXY", "GONOPROXY", "GONOSUMDB")
308 require.NoError(t, r.err, r.stderr)
310 var child map[string]string
311 require.NoError(t, json.Unmarshal([]byte(r.stdout), &child))
312 assert.Equal(t, "corp.example/*", child["GONOSUMDB"], "private paths must stay off the public checksum database")
313 assert.Equal(t, "none", child["GONOPROXY"])
314 assert.Equal(t, r.m.server.BaseURL(), child["GOPROXY"])
316 assert.Equal(t, "corp.example/*", b.overrides["GOPRIVATE"])
317 assert.Equal(t, "corp.example/*", b.overrides["GONOPROXY"])
318 assert.Equal(t, "corp.example/*", b.overrides["GONOSUMDB"])
319}
321// go env -changed prints only the settings that differ from the defaults, so
322// GOFLAGS=-changed would hide GOPROXY and the GOPRIVATE-derived GONOPROXY.
323func TestRunComposedGolangGOFLAGSChanged(t *testing.T) {
324 isolateGo(t, "GOPRIVATE=corp.example/*\n")
325 t.Setenv("GOFLAGS", "-changed -modcacherw")
326 t.Setenv("GLAB_DF_FAKE_DEFAULT", "allow")
327 b := newModuleBackend(t, "")
329 r := runRealGo(t, b, "env", "GOPROXY")
330 require.NoError(t, r.err, r.stderr)
332 assert.Contains(t, r.stdout, r.m.server.BaseURL(), "the child's GOPROXY must be the gate")
333 assert.Equal(t, "corp.example/*", b.overrides["GONOPROXY"], "private paths must not be fetched from the public proxy")
334 assert.Equal(t, "corp.example/*", b.overrides["GONOSUMDB"])
335}
Diagnostics7 git calls · 206 ms
CallStatusSizeTime0206 ms
discover200185 B4 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=2 git.response.bytes=59849 git.response.cache=HIT git.response.duration_ms=2 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0200361 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:02001.3 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=195 git.response.bytes=1327 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=2758711b27b3e124482ce287b5df86092bb20d62 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1327 git.pkt.payload.max=1300 git.sideband.band1.bytes=1300 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1300 git.pack.trailer=c83fc6004a9a1360845513c0fb16272b5c14a3e4Open pack map
fetch2003.7 KiB195 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3838 git.response.cache=MISS git.response.duration_ms=195 git.fetch.wants=46869877e3bbb720c38df9ca326d322df8812d18 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3838 git.pkt.payload.max=3811 git.sideband.band1.bytes=3811 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e1bd91d96f6c75
- Request
- 2976f0ea…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format