GitLabgitlab-org
cli
main
1package gomodproxy
3import (
4 "context"
5 "crypto/tls"
6 "crypto/x509"
7 "io"
8 "log/slog"
9 "maps"
10 "net/http"
11 "os"
12 "path/filepath"
13 "runtime"
14 "slices"
15 "strings"
17 "github.com/goproxy/goproxy"
19 "gitlab.com/gitlab-org/cli/internal/dbg"
20)
22// Backend serves allowed module proxy requests through goproxy and owns the
23// temp tree goproxy fetches into.
24type Backend struct {
25 handler http.Handler
26 tempDir string
27}
29// NewBackend builds a goproxy handler that fetches through goBin.
30// proxiedSumDBs lists the checksum databases served under /sumdb/, as
31// goproxy's "name" or "name URL" entries. envOverrides replaces the fetcher
32// environment's value for each key it sets, GOPROXY included, except the
33// isolation pins, which always win. upstreamCAs, PEM certificates trusted on
34// top of the system roots, covers goproxy's own requests and, where
35// newBackend allows it, the go subprocess.
36func NewBackend(goBin string, proxiedSumDBs []string, envOverrides map[string]string, upstreamCAs []byte) (*Backend, error) {
37 return newBackend(runtime.GOOS, goBin, proxiedSumDBs, envOverrides, upstreamCAs)
38}
40// newBackend skips the fetcher's SSL_CERT_FILE where it would drop the system
41// roots: on macOS and Windows, where since Go 1.27 it replaces the platform
42// store, and when only SSL_CERT_DIR is set, because go then reads neither its
43// default CA directories nor, with our file, its default CA file.
44func newBackend(goos, goBin string, proxiedSumDBs []string, envOverrides map[string]string, upstreamCAs []byte) (*Backend, error) {
45 tempDir, err := os.MkdirTemp("", "glab-df-gomodproxy-*")
46 if err != nil {
47 return nil, err
48 }
49 workDir := filepath.Join(tempDir, "work")
50 if err := os.Mkdir(workDir, 0o700); err != nil {
51 os.RemoveAll(tempDir)
52 return nil, err
53 }
55 env := fetcherEnv(tempDir, envOverrides)
56 var transport http.RoundTripper
57 if len(upstreamCAs) > 0 {
58 dropsSystemRoots := goos == "darwin" || goos == "windows" ||
59 (os.Getenv("SSL_CERT_DIR") != "" && os.Getenv("SSL_CERT_FILE") == "")
60 if dropsSystemRoots {
61 dbg.Debugf("gomodproxy: the go fetcher keeps its own CA trust; SSL_CERT_FILE would drop its system roots\n")
62 } else {
63 caFile, err := writeUpstreamCAs(tempDir, upstreamCAs)
64 if err != nil {
65 os.RemoveAll(tempDir)
66 return nil, err
67 }
68 env = append(env, "SSL_CERT_FILE="+caFile)
69 }
70 if transport, err = upstreamTransport(upstreamCAs); err != nil {
71 os.RemoveAll(tempDir)
72 return nil, err
73 }
74 }
75 return &Backend{
76 tempDir: tempDir,
77 handler: &goproxy.Goproxy{
78 Fetcher: &goFetcher{
79 goBin: goBin,
80 env: env,
81 dir: workDir,
82 },
83 Cacher: goproxy.DirCacher(filepath.Join(tempDir, "cache")),
84 TempDir: tempDir,
85 ProxiedSumDBs: proxiedSumDBs,
86 Transport: transport,
87 Logger: slog.New(debugHandler{}),
88 },
89 }, nil
90}
92// writeUpstreamCAs writes the bundle the go subprocess reads through
93// SSL_CERT_FILE. That replaces go's default CA file, but go still reads the
94// default CA directories or the user's SSL_CERT_DIR, so the roots go trusted
95// before stay trusted.
96func writeUpstreamCAs(tempDir string, upstreamCAs []byte) (string, error) {
97 path := filepath.Join(tempDir, "upstream-ca.pem")
98 return path, os.WriteFile(path, upstreamCAs, 0o600)
99}
101// upstreamTransport clones http.DefaultTransport so the custom pool keeps
102// http.ProxyFromEnvironment.
103func upstreamTransport(upstreamCAs []byte) (http.RoundTripper, error) {
104 pool, err := x509.SystemCertPool()
105 if err != nil {
106 return nil, err
107 }
108 pool.AppendCertsFromPEM(upstreamCAs)
109 t := http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert // net/http declares it as *http.Transport
110 t.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
111 return t, nil
112}
114func (b *Backend) ServeHTTP(w http.ResponseWriter, r *http.Request) {
115 failure := new(fetchFailure)
116 r = r.WithContext(context.WithValue(r.Context(), fetchFailureKey{}, failure))
117 b.handler.ServeHTTP(&failureWriter{ResponseWriter: w, failure: failure}, r)
118}
120// failureWriter turns goproxy's bare 500 for a failed fetch into a 502 that
121// carries go's error, which the child prints as the server response.
122type failureWriter struct {
123 http.ResponseWriter
124 failure *fetchFailure
125 replaced bool
126}
128func (w *failureWriter) WriteHeader(code int) {
129 if code != http.StatusInternalServerError || w.failure.msg == "" {
130 w.ResponseWriter.WriteHeader(code)
131 return
132 }
133 w.replaced = true
134 w.ResponseWriter.WriteHeader(http.StatusBadGateway)
135 _, _ = io.WriteString(w.ResponseWriter, w.failure.msg+"\n")
136}
138func (w *failureWriter) Write(p []byte) (int, error) {
139 if w.replaced {
140 return len(p), nil
141 }
142 return w.ResponseWriter.Write(p)
143}
145func (w *failureWriter) ReadFrom(r io.Reader) (int64, error) {
146 if w.replaced {
147 return io.Copy(io.Discard, r)
148 }
149 return io.Copy(w.ResponseWriter, r)
150}
152// Close removes the temp tree, including modules fetched while resolving a
153// request that was later blocked.
154func (b *Backend) Close() {
155 if err := os.RemoveAll(b.tempDir); err != nil {
156 dbg.Debugf("gomodproxy: removing %s: %v\n", b.tempDir, err)
157 }
158}
160// fetcherEnv starts from os.Environ because the fetcher's go needs PATH for
161// git and HOME for SSH keys, .netrc, gitconfig, and the go env file.
162// Settings are appended last because exec uses the last value per key. The
163// isolation pins are applied after the overrides:
164// - a private GOMODCACHE, because sharing the child's deadlocks on the
165// module-cache lock and would let the child skip the zip request;
166// - -modcacherw, so Close can delete what go extracts read-only;
167// - GOTOOLCHAIN=local, so the fetcher never downloads a toolchain;
168// - GOWORK=off, so an absolute GOWORK cannot apply workspace replaces;
169// - GO111MODULE=on, because go list -m and go mod download refuse to run
170// with modules off.
171func fetcherEnv(tempDir string, overrides map[string]string) []string {
172 settings := map[string]string{}
173 maps.Copy(settings, overrides)
174 maps.Copy(settings, map[string]string{
175 "GO111MODULE": "on",
176 "GOMODCACHE": filepath.Join(tempDir, "modcache"),
177 "GOFLAGS": "-mod=mod -modcacherw",
178 "GOTOOLCHAIN": "local",
179 "GOWORK": "off",
180 })
182 env := os.Environ()
183 for _, k := range slices.Sorted(maps.Keys(settings)) {
184 env = append(env, k+"="+settings[k])
185 }
186 return env
187}
189// debugHandler routes goproxy's slog output to glab's debug log; on stderr
190// it would corrupt the firewall summary.
191type debugHandler struct{}
193func (debugHandler) Enabled(context.Context, slog.Level) bool { return true }
195func (debugHandler) Handle(_ context.Context, r slog.Record) error {
196 var b strings.Builder
197 b.WriteString(r.Message)
198 r.Attrs(func(a slog.Attr) bool {
199 b.WriteString(" " + a.String())
200 return true
201 })
202 dbg.Debugf("gomodproxy: %s\n", redactUserinfo(b.String()))
203 return nil
204}
206func (h debugHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
208func (h debugHandler) WithGroup(string) slog.Handler { return h }
Diagnostics7 git calls · 254 ms
CallStatusSizeTime0254 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:0200379 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=242 git.response.bytes=379 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=21702eb6df99111b4d7a039e655c720d4209d5b6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=379 git.pkt.payload.max=352 git.sideband.band1.bytes=352 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=352 git.pack.trailer=7d54607262a7ed73d2686c4df38d0272960c21fcOpen pack map
fetch2002.8 KiB242 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2875 git.response.cache=MISS git.response.duration_ms=242 git.fetch.wants=a04fd3077ac2c87e57999228e46f80d95debf3d4 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2875 git.pkt.payload.max=2848 git.sideband.band1.bytes=2848 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e1a9c45f9e2c38
- Request
- 317a91c4…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format