GitLabgitlab-org

cli

main
ply7 git calls197 ms100 KiB5 cachedprotocol v2
340 lines12 KiBRaw
1// Package session supports the job-level Dependency Firewall proxy
2// (`glab dependency-firewall job-proxy`). Unlike the per-command package managers in
3// package pm — each of which starts a proxy for the length of one command — a
4// session runs one inspection proxy for a whole CI job and emits the
5// environment that routes every tool through it and makes every tool trust it,
6// without clobbering a caller's existing proxy or CA configuration.
7package session
9import (
10 "crypto/x509"
11 "encoding/pem"
12 "fmt"
13 "maps"
14 "os"
15 "os/exec"
16 "path/filepath"
17 "sort"
18 "strings"
20 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/pm"
23// routingManagers are the package managers whose own routing variables the
24// session sets, so a manager that does not simply follow HTTP(S)_PROXY (pip
25// reads PIP_PROXY, npm reads NPM_CONFIG_*, bundler reads BUNDLE_*) is still
26// routed through the proxy. Trust is supplied once, additively, by trustVars —
27// not per manager — so the managers' CATrustEnviron is deliberately not called
28// here. Maven and Gradle are omitted: routing them needs a generated
29// settings.xml and JVM trust store, which JVMTrustEnv writes instead of an
30// environment variable.
31func routingManagers() []pm.PackageManager {
32 return []pm.PackageManager{
33 pm.NPM(), pm.Pnpm(), pm.Yarn(),
34 pm.Pip(), pm.Pipenv(), pm.Poetry(), pm.Uv(),
35 pm.Bundle(), pm.Gem(),
36 pm.Cargo(),
37 }
40// BuildJobEnv returns the environment, as "KEY=value" strings, that routes a
41// job's tools through the proxy at proxyURL and makes them trust its MITM CA.
42//
43// Routing points HTTP(S)_PROXY (and the per-manager routing variables) at the
44// proxy, and preserves the caller's NO_PROXY so their internal hosts still go
45// direct, adding only loopback so local traffic is never proxied.
46//
47// Trust is additive: mergedBundlePath holds the system roots, the caller's
48// existing anchors and the proxy CA, so pointing a tool's CA variable at it
49// never drops a CA the tool trusted before. Installing the proxy CA into the
50// system trust store — done separately by the command — covers everything
51// that reads the system store (git, wget, OpenSSL, apt).
52func BuildJobEnv(proxyURL, mergedBundlePath, callerNoProxy string) []string {
53 vars := map[string]string{}
55 // Routing. Both cases are set because tools disagree on which they read.
56 // HTTP_PROXY is set too, so a plain-HTTP registry is checked rather than
57 // fetched directly; the proxy forwards other plain-HTTP traffic unchanged.
58 for _, k := range []string{"HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"} {
59 vars[k] = proxyURL
60 }
61 noProxy := mergeNoProxy(callerNoProxy)
62 vars["NO_PROXY"] = noProxy
63 vars["no_proxy"] = noProxy
65 for _, m := range routingManagers() {
66 for _, kv := range m.Environment(proxyURL) {
67 if k, v, ok := strings.Cut(kv, "="); ok {
68 vars[k] = v
69 }
70 }
71 }
73 // Trust, all additive.
74 maps.Copy(vars, trustVars(mergedBundlePath))
76 env := make([]string, 0, len(vars))
77 for k, v := range vars {
78 env = append(env, k+"="+v)
79 }
80 sort.Strings(env)
81 return env
84// trustVars are the only trust variables the job gets. Each points at the
85// merged bundle, which adds the proxy CA to what a tool already trusted (see
86// WriteMergedBundle) rather than replacing it.
87func trustVars(mergedBundlePath string) map[string]string {
88 return map[string]string{
89 // Node adds these anchors to its built-in roots. The merged bundle keeps
90 // a caller's own NODE_EXTRA_CA_CERTS, which this replaces. NODE_USE_SYSTEM_CA
91 // makes newer Node also honor the system store the command installs into.
92 "NODE_EXTRA_CA_CERTS": mergedBundlePath,
93 "NODE_USE_SYSTEM_CA": "1",
94 // requests/certifi ignore the system store, so they get the merged
95 // bundle; pip reads REQUESTS_CA_BUNDLE and also honors PIP_CERT.
96 "REQUESTS_CA_BUNDLE": mergedBundlePath,
97 "PIP_CERT": mergedBundlePath,
98 // uv verifies with the system store rather than its bundled roots.
99 "UV_NATIVE_TLS": "true",
100 // cargo and git read their own CA file variables.
101 "CARGO_HTTP_CAINFO": mergedBundlePath,
102 "GIT_SSL_CAINFO": mergedBundlePath,
103 // curl prefers CURL_CA_BUNDLE over the system store, and without it curl
104 // trusts the proxy only if the system-store install (root only) worked.
105 "CURL_CA_BUNDLE": mergedBundlePath,
106 // RubyGems reads OpenSSL's SSL_CERT_FILE and Bundler its own variable;
107 // without them, gem and bundle trust the proxy only if the system-store
108 // install succeeded, which needs root.
109 "SSL_CERT_FILE": mergedBundlePath,
110 "BUNDLE_SSL_CA_CERT": mergedBundlePath,
111 }
114// mergeNoProxy keeps the caller's NO_PROXY entries and ensures loopback is
115// present, so the job's own local traffic is never routed through the proxy
116// while the caller's internal hosts still go direct.
117func mergeNoProxy(callerNoProxy string) string {
118 seen := map[string]struct{}{}
119 var out []string
120 add := func(s string) {
121 s = strings.TrimSpace(s)
122 if s == "" {
123 return
124 }
125 if _, dup := seen[s]; dup {
126 return
127 }
128 seen[s] = struct{}{}
129 out = append(out, s)
130 }
131 for e := range strings.SplitSeq(callerNoProxy, ",") {
132 add(e)
133 }
134 for _, e := range []string{"localhost", "127.0.0.1", "::1"} {
135 add(e)
136 }
137 return strings.Join(out, ",")
140// WriteMergedBundle writes a CA bundle containing the system roots, any
141// caller-supplied CA bundles (userBundlePaths), and the proxy CA at
142// proxyCAPath, to stateDir/ca-bundle.pem, and returns its path. Tools that
143// replace their trust from a single bundle (Python requests, cargo) can be
144// pointed at it without losing the roots they had, which is what makes the
145// proxy's trust additive rather than a replacement. A system or caller bundle
146// that can't be read is reported through logf, because the tools pointed at
147// the merged bundle lose those roots.
148func WriteMergedBundle(stateDir, proxyCAPath string, userBundlePaths []string, logf func(string, ...any)) (string, error) {
149 var bundle []byte
150 appendPEM := func(b []byte) {
151 if len(b) == 0 {
152 return
153 }
154 bundle = append(bundle, b...)
155 if bundle[len(bundle)-1] != '\n' {
156 bundle = append(bundle, '\n')
157 }
158 }
160 foundSystem := false
161 for _, p := range pm.SystemCABundlePaths {
162 if b, err := os.ReadFile(p); err == nil && len(b) > 0 {
163 appendPEM(b)
164 foundSystem = true
165 break
166 }
167 }
168 if !foundSystem {
169 logf("dependency firewall: no system CA bundle found; tools that use the merged bundle will trust only the proxy and your own CA bundles\n")
170 }
171 seen := map[string]struct{}{}
172 for _, p := range userBundlePaths {
173 if p == "" {
174 continue
175 }
176 if _, dup := seen[p]; dup {
177 continue
178 }
179 seen[p] = struct{}{}
180 b, err := os.ReadFile(p)
181 if err != nil {
182 logf("dependency firewall: could not read CA bundle %s, so it is left out of the merged bundle: %v\n", p, err)
183 continue
184 }
185 appendPEM(b)
186 }
187 if b, err := os.ReadFile(proxyCAPath); err == nil {
188 appendPEM(b)
189 } else {
190 return "", fmt.Errorf("reading proxy CA at %s: %w", proxyCAPath, err)
191 }
193 path := filepath.Join(stateDir, "ca-bundle.pem")
194 if err := os.WriteFile(path, bundle, 0o644); err != nil {
195 return "", err
196 }
197 return path, nil
200// writeCAPEM writes the proxy CA certificate to path as a single PEM block.
201func writeCAPEM(path string, ca *x509.Certificate) error {
202 return os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.Raw}), 0o644)
205// callerBundleEnvVars names the environment variables whose values point at a
206// caller's existing CA bundle, so WriteMergedBundle can fold those anchors in.
207var callerBundleEnvVars = []string{
208 "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", "PIP_CERT",
209 "NODE_EXTRA_CA_CERTS", "CURL_CA_BUNDLE", "GIT_SSL_CAINFO", "CARGO_HTTP_CAINFO",
210 "BUNDLE_SSL_CA_CERT",
213// CallerBundlePaths returns the non-empty values of callerBundleEnvVars from
214// env ("KEY=value" strings), for seeding WriteMergedBundle.
215func CallerBundlePaths(env []string) []string {
216 want := map[string]struct{}{}
217 for _, k := range callerBundleEnvVars {
218 want[k] = struct{}{}
219 }
220 var out []string
221 for _, e := range env {
222 if k, v, ok := strings.Cut(e, "="); ok && v != "" {
223 if _, ok := want[k]; ok {
224 out = append(out, v)
225 }
226 }
227 }
228 return out
231// systemTrustTarget is an OS trust-store anchor directory and the tool that
232// rebuilds the store after a certificate is added to or removed from it.
233type systemTrustTarget struct {
234 anchorDir string
235 tool string
236 toolArgs []string
239func systemTrustTargets() []systemTrustTarget {
240 return []systemTrustTarget{
241 {anchorDir: "/usr/local/share/ca-certificates", tool: "update-ca-certificates"}, // Debian, Ubuntu, Alpine
242 {anchorDir: "/etc/pki/ca-trust/source/anchors", tool: "update-ca-trust", toolArgs: []string{"extract"}}, // Fedora, RHEL
243 }
246// systemTrustAnchorName is the per-run file the proxy CA is staged under in an
247// anchor directory. It includes the state dir's base name (unique per run) so
248// concurrent jobs on one host do not overwrite each other's CA, and so stop can
249// remove exactly the file its own start added instead of a shared fixed name.
250func systemTrustAnchorName(stateDir string) string {
251 return "glab-df-proxy-" + filepath.Base(stateDir) + ".crt"
254// InstallSystemTrust installs the proxy CA at proxyCAPath into the OS trust
255// store, so tools that read it (curl, git, wget, OpenSSL, apt) trust the proxy
256// without any per-tool variable. It is best-effort: on an unsupported image or
257// without privilege it logs and returns, leaving the env-var bundles as the
258// fallback. It reports whether the install succeeded.
259func InstallSystemTrust(stateDir, proxyCAPath string, logf func(string, ...any)) bool {
260 for _, t := range systemTrustTargets() {
261 if !dirExists(t.anchorDir) {
262 continue
263 }
264 toolPath, err := exec.LookPath(t.tool)
265 if err != nil {
266 continue
267 }
268 dst := filepath.Join(t.anchorDir, systemTrustAnchorName(stateDir))
269 if err := copyFile(proxyCAPath, dst); err != nil {
270 logf("dependency firewall: could not stage proxy CA at %s: %v\n", dst, err)
271 continue
272 }
273 if out, err := exec.Command(toolPath, t.toolArgs...).CombinedOutput(); err != nil {
274 logf("dependency firewall: %s failed: %v: %s\n", t.tool, err, strings.TrimSpace(string(out)))
275 continue
276 }
277 logf("dependency firewall: installed proxy CA into the system trust store via %s\n", t.tool)
278 return true
279 }
280 logf("dependency firewall: could not install the proxy CA into a system trust store; tools that only read it (for example wget and apt) may not trust the proxy\n")
281 return false
284// RemoveSystemTrust removes the per-run proxy CA that InstallSystemTrust staged
285// for stateDir from every anchor directory and rebuilds the store, so a job's CA
286// does not linger in the host trust store after stop. It is best-effort and, like
287// InstallSystemTrust, stays silent when there was nothing to remove.
288func RemoveSystemTrust(stateDir string, logf func(string, ...any)) {
289 for _, t := range systemTrustTargets() {
290 if !dirExists(t.anchorDir) {
291 continue
292 }
293 dst := filepath.Join(t.anchorDir, systemTrustAnchorName(stateDir))
294 if err := os.Remove(dst); err != nil {
295 if !os.IsNotExist(err) {
296 logf("dependency firewall: could not remove proxy CA at %s: %v\n", dst, err)
297 }
298 continue
299 }
300 toolPath, err := exec.LookPath(t.tool)
301 if err != nil {
302 continue
303 }
304 if out, err := exec.Command(toolPath, t.toolArgs...).CombinedOutput(); err != nil {
305 logf("dependency firewall: %s failed: %v: %s\n", t.tool, err, strings.TrimSpace(string(out)))
306 }
307 }
310func dirExists(p string) bool {
311 fi, err := os.Stat(p)
312 return err == nil && fi.IsDir()
315func copyFile(src, dst string) error {
316 b, err := os.ReadFile(src)
317 if err != nil {
318 return err
319 }
320 return os.WriteFile(dst, b, 0o644)
323// ShellExports renders env ("KEY=value" strings) as POSIX `export` lines that a
324// shell can eval. Values are single-quoted with embedded single quotes escaped,
325// so a value with spaces or shell metacharacters is passed through verbatim.
326func ShellExports(env []string) string {
327 var b strings.Builder
328 for _, e := range env {
329 k, v, ok := strings.Cut(e, "=")
330 if !ok {
331 continue
332 }
333 fmt.Fprintf(&b, "export %s=%s\n", k, singleQuote(v))
334 }
335 return b.String()
338func singleQuote(s string) string {
339 return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
Diagnostics7 git calls · 197 ms
CallStatusSizeTime0197 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:0200279 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=188
git.response.bytes=279
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=97ca01ef53be93b101ee1321413cdbf4a0c86c70
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=279
git.pkt.payload.max=252
git.sideband.band1.bytes=252
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=252
git.pack.trailer=bfb89af7a897fc72572aa6703a2786499c09a4f4
Open pack map
fetch2004.4 KiB188 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=4523
git.response.cache=MISS
git.response.duration_ms=188
git.fetch.wants=29068a0e7c79e78ddaed7268fa0f130ef8f925d7
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4523
git.pkt.payload.max=4496
git.sideband.band1.bytes=4496
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e25a1e8cdba23e
Request
4798d7fe…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format