GitLabgitlab-org
cli
main
1package gomodproxy
3import (
4 "context"
5 "crypto/rand"
6 "crypto/subtle"
7 "errors"
8 "net"
9 "net/http"
10 "net/url"
11 "path"
12 "runtime/debug"
13 "slices"
14 "strings"
15 "sync"
16 "sync/atomic"
17 "time"
19 "gitlab.com/gitlab-org/cli/internal/dbg"
20 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
21 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/verdict"
22)
24const (
25 drainTimeout = 5 * time.Second
26 readHeaderTimeout = 30 * time.Second
27)
29// Server serves the Go module proxy protocol on loopback and gates module
30// zip downloads against the Dependency Firewall.
31//
32// The backend's fetcher carries the user's version-control credentials, so
33// only a client that knows the per-run path token and addresses the listener
34// by its own address reaches it: that excludes other local processes and
35// DNS-rebinding web pages.
36type Server struct {
37 checker policy.Checker
38 projectID string
39 backend http.Handler
40 token string
42 ctx context.Context
43 cancel context.CancelFunc
45 listener net.Listener
46 addr string
47 server *http.Server
49 checked atomic.Int64
51 mu sync.Mutex
52 verdicts []verdict.Entry
53 seen map[string]struct{}
55 stopOnce sync.Once
56}
58func New(checker policy.Checker, projectID string, backend http.Handler) *Server {
59 s := &Server{
60 checker: checker,
61 projectID: projectID,
62 backend: backend,
63 token: rand.Text(),
64 seen: make(map[string]struct{}),
65 }
66 s.ctx, s.cancel = context.WithCancel(context.Background())
67 return s
68}
70// Start listens on an ephemeral loopback port and begins serving.
71func (s *Server) Start() error {
72 ln, err := net.Listen("tcp", "127.0.0.1:0")
73 if err != nil {
74 return err
75 }
76 s.listener = ln
77 s.addr = ln.Addr().String()
78 s.server = &http.Server{
79 Handler: http.HandlerFunc(s.handle),
80 ReadHeaderTimeout: readHeaderTimeout,
81 BaseContext: func(net.Listener) context.Context { return s.ctx },
82 }
83 go func() {
84 if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
85 dbg.Debugf("gomodproxy: stopped serving: %v\n", err)
86 }
87 }()
88 return nil
89}
91// BaseURL is the value to hand the child as GOPROXY. It is empty before Start.
92func (s *Server) BaseURL() string {
93 if s.addr == "" {
94 return ""
95 }
96 return "http://" + s.addr + "/" + s.token
97}
99// Stop cancels in-flight policy checks and fetches, waits up to drainTimeout
100// for their handlers to return, then force-closes what is left. Waiting
101// matters because the caller removes the backend's temp tree next. Stop is
102// idempotent.
103func (s *Server) Stop() {
104 s.stopOnce.Do(func() {
105 s.cancel()
106 if s.server == nil {
107 return
108 }
109 ctx, cancel := context.WithTimeout(context.Background(), drainTimeout)
110 defer cancel()
111 if err := s.server.Shutdown(ctx); err != nil {
112 dbg.Debugf("gomodproxy: drain: %v\n", err)
113 _ = s.server.Close()
114 }
115 })
116}
118// Verdicts returns a copy of the blocked and warned verdicts recorded so far.
119func (s *Server) Verdicts() []verdict.Entry {
120 s.mu.Lock()
121 defer s.mu.Unlock()
122 return slices.Clone(s.verdicts)
123}
125// CheckedZips reports how many module zips got a policy decision.
126func (s *Server) CheckedZips() int {
127 return int(s.checked.Load())
128}
130func (s *Server) record(e verdict.Entry) {
131 s.mu.Lock()
132 defer s.mu.Unlock()
133 key := e.Key()
134 if _, ok := s.seen[key]; ok {
135 return
136 }
137 s.seen[key] = struct{}{}
138 s.verdicts = append(s.verdicts, e)
139}
141// handle refuses anything it cannot classify with a 404, which is safe only
142// because GOPROXY names this server alone: there is no next source for go to
143// fall through to.
144func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
145 rest, ok := s.authorize(r)
146 if !ok {
147 http.NotFound(w, r)
148 return
149 }
151 if r.Method != http.MethodGet && r.Method != http.MethodHead {
152 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
153 return
154 }
156 // go percent-encodes only the '!' of the module case encoding
157 // (github.com/BurntSushi -> github.com/%21burnt%21sushi).
158 reqPath := strings.ReplaceAll("/"+rest, "%21", "!")
160 // The gate checks the escaped path while goproxy routes the decoded one,
161 // and module.UnescapeVersion accepts a literal '%'. Any other escape would
162 // let ".../@v/v0%2e9%2e1.zip" be checked as an unknown version and then
163 // served as v0.9.1.
164 decoded, err := url.PathUnescape(reqPath)
165 if err != nil || decoded != reqPath {
166 http.NotFound(w, r)
167 return
168 }
170 if path.Clean(reqPath) != reqPath {
171 http.NotFound(w, r)
172 return
173 }
175 if isSumDB(reqPath) {
176 s.delegate(w, r, reqPath)
177 return
178 }
180 req, ok := parseRequest(reqPath)
181 if !ok {
182 http.NotFound(w, r)
183 return
184 }
186 // Metadata passes ungated: go probes every path prefix of a module
187 // ("/github.com/@v/v1.6.0.info"), and a 403 on one of those would break
188 // resolution for every module under it.
189 if req.kind != kindZip || isToolchain(req) {
190 s.delegate(w, r, reqPath)
191 return
192 }
194 res, decided := s.check(req) //nolint:contextcheck // a client disconnect must not cancel the check; see check
195 if !decided {
196 http.Error(w, "dependency firewall stopped", http.StatusServiceUnavailable)
197 return
198 }
199 switch {
200 case res.Blocked():
201 s.record(verdict.Entry{
202 Package: req.module,
203 Version: req.version,
204 Verdict: verdict.Blocked,
205 Reason: res.Reason,
206 Status: http.StatusForbidden,
207 Timestamp: time.Now().UTC().Format(time.RFC3339),
208 })
209 reason := res.Reason
210 if res.Internal {
211 reason = policy.InternalErrorPhrase(policy.Coordinate{Name: req.module, Version: req.version})
212 }
213 writeBlocked(w, req.module, req.version, reason)
214 return
215 case res.Warned():
216 s.record(verdict.Entry{
217 Package: req.module,
218 Version: req.version,
219 Verdict: verdict.Warning,
220 Reason: res.Reason,
221 Timestamp: time.Now().UTC().Format(time.RFC3339),
222 })
223 }
224 s.delegate(w, r, reqPath)
225}
227func (s *Server) authorize(r *http.Request) (string, bool) {
228 if r.Host != s.addr {
229 return "", false
230 }
231 token, rest, ok := strings.Cut(strings.TrimPrefix(r.URL.EscapedPath(), "/"), "/")
232 if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(s.token)) != 1 {
233 return "", false
234 }
235 return rest, true
236}
238// check fails closed on any error so a firewall outage cannot admit a
239// package. It reports false when Stop ended the check: that is not a
240// firewall decision, and recording it as Blocked would fail a later
241// ci-summary for a module nobody blocked.
242//
243// The check runs under the server context rather than the request's, which
244// net/http cancels as soon as the client disconnects. It runs in its own
245// goroutine because policy.CachingChecker calls the API under
246// context.WithoutCancel, so Stop could otherwise neither end it nor stop it
247// recording a verdict after Stop returned.
248func (s *Server) check(req request) (policy.Result, bool) {
249 ctx, cancel := context.WithTimeout(s.ctx, policy.CheckTimeout)
250 defer cancel()
252 type outcome struct {
253 res policy.Result
254 err error
255 }
256 done := make(chan outcome, 1)
257 go func() {
258 // This goroutine is outside net/http's recover, so a panicking checker
259 // would end glab. Block instead; the value stays out of the reason.
260 defer func() {
261 if r := recover(); r != nil {
262 dbg.Debugf("dependency firewall policy check panicked for %s@%s: %v\n%s", req.module, req.version, r, debug.Stack())
263 done <- outcome{err: policy.ErrCheckPanicked}
264 }
265 }()
266 res, err := s.checker.Check(ctx, policy.Request{
267 Coordinate: policy.Coordinate{
268 Ecosystem: "golang",
269 Name: req.module,
270 Version: req.version,
271 },
272 ProjectID: s.projectID,
273 Operation: policy.Download,
274 })
275 done <- outcome{res, err}
276 }()
278 var out outcome
279 select {
280 case out = <-done:
281 case <-s.ctx.Done():
282 return policy.Result{}, false
283 }
284 if out.err != nil && s.ctx.Err() != nil {
285 return policy.Result{}, false
286 }
287 s.checked.Add(1)
288 if errors.Is(out.err, policy.ErrCheckPanicked) {
289 return policy.InternalErrorResult(policy.Coordinate{Name: req.module, Version: req.version}), true
290 }
291 if out.err != nil {
292 return policy.Result{
293 Verdict: verdict.Blocked,
294 Reason: "policy check failed: " + out.err.Error(),
295 }, true
296 }
297 return out.res, true
298}
300func (s *Server) delegate(w http.ResponseWriter, r *http.Request, reqPath string) {
301 out := new(http.Request)
302 *out = *r
303 out.URL = new(url.URL)
304 *out.URL = *r.URL
305 out.URL.Path = reqPath
306 out.URL.RawPath = ""
307 out.RequestURI = ""
308 s.backend.ServeHTTP(w, out)
309}
Diagnostics7 git calls · 222 ms
CallStatusSizeTime0222 ms
discover200185 B4 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchtree:0200361 B · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=6 git.response.bytes=361 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=361 git.pkt.payload.max=334 git.sideband.band1.bytes=334 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=334 git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32Open pack map
fetchtree:0200379 B · hit6 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=206 git.response.bytes=379 git.response.cache=HIT git.response.duration_ms=6 git.fetch.filter=tree:0 git.fetch.wants=21702eb6df99111b4d7a039e655c720d4209d5b6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=379 git.pkt.payload.max=352 git.sideband.band1.bytes=352 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=352 git.pack.trailer=7d54607262a7ed73d2686c4df38d0272960c21fcOpen pack map
fetch2003.3 KiB206 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=3412 git.response.cache=MISS git.response.duration_ms=206 git.fetch.wants=08b671a275ae0a1113d697710cccdbce9be28262 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=3412 git.pkt.payload.max=3385 git.sideband.band1.bytes=3385 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e235a86830a594
- Request
- bdfb3c07…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format