GitLabgitlab-org

cli

main
ply8 git calls329 ms100 KiB5 cachedprotocol v2
290 lines7.8 KiBRaw
1package graph
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log/slog"
10 "mime/multipart"
11 "net"
12 "net/http"
13 "net/textproto"
15 "github.com/coder/websocket"
16 "github.com/d2lang/d2/d2format"
17 "github.com/d2lang/d2/d2graph"
18 "github.com/d2lang/d2/d2layouts/d2dagrelayout"
19 "github.com/d2lang/d2/d2lib"
20 "github.com/d2lang/d2/d2renderers/d2svg"
21 "github.com/d2lang/d2/lib/log"
22 "github.com/d2lang/d2/lib/textmeasure"
23 "github.com/pkg/browser"
25 "gitlab.com/gitlab-org/cli/internal/iostreams"
28const (
29 graphAPISubprotocolName = "gitlab-agent-graph-api"
32type server struct {
33 log *slog.Logger
34 io *iostreams.IOStreams
35 httpClient *http.Client
36 graphAPIURL string
37 listenNet, listenAddr string
38 authorization string
39 watchRequest []byte
42func (s *server) Run(ctx context.Context) error {
43 var lc net.ListenConfig
44 l, err := lc.Listen(ctx, s.listenNet, s.listenAddr)
45 if err != nil {
46 return err
47 }
48 s.io.LogInfof("Listening on http://%s\n", l.Addr())
49 srv := &http.Server{
50 Handler: http.HandlerFunc(s.handle),
51 }
52 err = browser.OpenURL(fmt.Sprintf("http://%s", l.Addr()))
53 if err != nil {
54 s.io.LogError("Failed to open browser:", err)
55 }
56 // Close rather than Shutdown: the graph handler holds a long-lived stream that Shutdown would wait on.
57 stop := context.AfterFunc(ctx, func() { _ = srv.Close() })
58 defer stop()
59 err = srv.Serve(l)
60 if errors.Is(err, http.ErrServerClosed) {
61 err = nil
62 }
63 return err
66func (s *server) handle(w http.ResponseWriter, r *http.Request) {
67 ctx, cancel := context.WithCancel(r.Context())
68 defer cancel()
69 conn, _, err := websocket.Dial(ctx, s.graphAPIURL, &websocket.DialOptions{ //nolint:bodyclose // upgrade response body is consumed by the websocket library
70 HTTPClient: s.httpClient,
71 HTTPHeader: http.Header{
72 "Authorization": []string{s.authorization},
73 },
74 Subprotocols: []string{graphAPISubprotocolName},
75 })
76 if err != nil {
77 s.reportError(w, fmt.Errorf("WebSocket dial: %w", err), false)
78 return
79 }
80 conn.SetReadLimit(4 * 1024 * 1024) // 1MiB is Kubernetes max object size (that'd be the minimum).
82 err = conn.Write(ctx, websocket.MessageText, s.watchRequest)
83 if err != nil {
84 s.reportError(w, fmt.Errorf("WebSocket write: %w", err), false)
85 return
86 }
87 ctx = log.With(ctx, s.log) // logger for D2
88 graphSrcC, readyC := s.buildGraph(ctx, s.readActions(ctx, conn))
89 s.renderAndWrite(ctx, w, graphSrcC, readyC)
92func (s *server) readActions(ctx context.Context, conn *websocket.Conn) <-chan dataError[jsonWatchGraphResponse] {
93 resp := make(chan dataError[jsonWatchGraphResponse])
94 go func() {
95 done := ctx.Done()
96 for {
97 mt, data, err := conn.Read(ctx)
98 if err != nil {
99 sendErr(done, resp, fmt.Errorf("WebSocket read: %w", err))
100 return
101 }
102 if mt != websocket.MessageText { // shouldn't ever happen
103 sendErr(done, resp, errors.New("unexpected message type"))
104 return
105 }
106 var d jsonWatchGraphResponse
107 err = json.Unmarshal(data, &d)
108 if err != nil {
109 sendErr(done, resp, fmt.Errorf("JSON unmarshal: %w", err))
110 return
111 }
112 sendData(done, resp, d)
113 }
114 }()
115 return resp
118func (s *server) buildGraph(ctx context.Context, srcC <-chan dataError[jsonWatchGraphResponse]) (<-chan dataError[string], chan<- struct{}) {
119 resp := make(chan dataError[string], 1) // enable non-blocking send
120 readyC := make(chan struct{}, 1) // enable non-blocking send
121 go func() {
122 done := ctx.Done()
123 b, err := newGraphBuilder(ctx, s.io)
124 if err != nil {
125 sendErr(done, resp, fmt.Errorf("newGraphBuilder: %w", err))
126 return
127 }
128 producerReady := false
129 consumerReady := false
130 lastSentAST := ""
131 for {
132 select {
133 case <-done:
134 return
135 case <-readyC:
136 consumerReady = true
137 case src := <-srcC:
138 if src.err != nil {
139 sendErr(done, resp, src.err)
140 return
141 }
142 s.logWarnings(src.data.Warnings)
143 if src.data.Error != nil {
144 sendErr(done, resp, src.data.Error)
145 return
146 }
147 err = b.applyActions(src.data.Actions)
148 if err != nil {
149 sendErr(done, resp, err)
150 return
151 }
152 producerReady = true
153 }
154 if consumerReady && producerReady {
155 newAST := d2format.Format(b.g.AST)
156 if newAST == lastSentAST {
157 continue // no changes, skip sending.
158 }
159 resp <- dataError[string]{data: newAST}
160 consumerReady = false
161 producerReady = false
162 lastSentAST = newAST
163 }
164 }
165 }()
166 return resp, readyC
169// must run on the http server's handler goroutine since can panic.
170func (s *server) renderAndWrite(ctx context.Context, w http.ResponseWriter, srcC <-chan dataError[string], readyC chan<- struct{}) {
171 ruler, err := textmeasure.NewRuler()
172 if err != nil {
173 s.reportError(w, fmt.Errorf("textmeasure.NewRuler: %w", err), false)
174 return
175 }
176 flush := http.NewResponseController(w).Flush
177 header := textproto.MIMEHeader{
178 "Content-Type": []string{"image/svg+xml"},
179 }
180 mw := multipart.NewWriter(w)
181 done := ctx.Done()
182 var part io.Writer
183 nilableReadyC := readyC
184 for {
185 select {
186 case <-done:
187 panic(http.ErrAbortHandler)
188 case nilableReadyC <- struct{}{}: // let the producer know we are ready to receive the next frame
189 nilableReadyC = nil // ok, producer knows we are ready, we can stop poking it for now.
190 case src := <-srcC:
191 nilableReadyC = readyC // re-enable the above case
192 if src.err != nil {
193 s.reportError(w, src.err, part != nil)
194 return
195 }
196 svgData, err := renderSVG(ctx, ruler, src.data)
197 if err != nil {
198 s.reportError(w, err, part != nil)
199 return
200 }
201 if part == nil { // first part
202 h := w.Header()
203 h.Set("Content-Type", "multipart/x-mixed-replace;boundary="+mw.Boundary())
204 h.Set("Transfer-Encoding", "identity")
205 w.WriteHeader(http.StatusOK)
206 part, err = mw.CreatePart(header)
207 if err != nil {
208 s.reportError(w, fmt.Errorf("CreatePart: %w", err), true)
209 return
210 }
211 }
213 _, err = part.Write(svgData)
214 if err != nil {
215 s.reportError(w, err, true)
216 return
217 }
218 part, err = mw.CreatePart(header)
219 if err != nil {
220 s.reportError(w, fmt.Errorf("CreatePart: %w", err), true)
221 return
222 }
223 err = flush()
224 if err != nil {
225 s.reportError(w, err, true)
226 return
227 }
228 }
229 }
232// renderSVG compiles D2 source with the dagre layout and renders it to SVG.
233func renderSVG(ctx context.Context, ruler *textmeasure.Ruler, src string) ([]byte, error) {
234 renderOpts := &d2svg.RenderOpts{}
235 diagram, _, err := d2lib.Compile(ctx, src, &d2lib.CompileOptions{
236 Ruler: ruler,
237 LayoutResolver: func(string) (d2graph.LayoutGraph, error) {
238 return d2dagrelayout.DefaultLayout, nil
239 },
240 }, renderOpts)
241 if err != nil {
242 return nil, fmt.Errorf("d2lib.Compile: %w", err)
243 }
244 svg, err := d2svg.Render(diagram, renderOpts)
245 if err != nil {
246 return nil, fmt.Errorf("d2svg.Render: %w", err)
247 }
248 return svg, nil
251func (s *server) logWarnings(w []jsonWatchGraphWarning) {
252 for _, warning := range w {
253 if len(warning.Attributes) > 0 {
254 s.io.LogInfof("Warning: %s: %s (%v)\n", warning.Type, warning.Message, warning.Attributes)
255 } else {
256 s.io.LogInfof("Warning: %s: %s\n", warning.Type, warning.Message)
257 }
258 }
261func (s *server) reportError(w http.ResponseWriter, err error, dataWritten bool) {
262 s.io.LogError(err.Error())
263 if dataWritten {
264 // we've written something already, the only way to let the caller know there was an issue is to
265 // drop the connection.
266 panic(http.ErrAbortHandler)
267 } else {
268 http.Error(w, err.Error(), http.StatusInternalServerError)
269 }
272type dataError[T any] struct {
273 data T
274 err error
277func sendErr[T any](done <-chan struct{}, resp chan<- dataError[T], err error) {
278 sendDataOrErr(done, resp, dataError[T]{err: err})
281func sendData[T any](done <-chan struct{}, resp chan<- dataError[T], data T) {
282 sendDataOrErr(done, resp, dataError[T]{data: data})
285func sendDataOrErr[T any](done <-chan struct{}, resp chan<- dataError[T], v dataError[T]) {
286 select {
287 case <-done:
288 case resp <- v:
289 }
Diagnostics8 git calls · 329 ms
CallStatusSizeTime0329 ms
discover200185 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchtree:02001.8 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=1857
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1857
git.pkt.payload.max=1830
git.sideband.band1.bytes=1830
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1830
git.pack.trailer=fb03183b7909876778f740d196d7516811003517
Open pack map
fetchtree:0200200 B · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=200
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=9988d47506dcbe3004b8b72cbdb9d66d6d75cda4
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=200
git.pkt.payload.max=173
git.sideband.band1.bytes=173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=173
git.pack.trailer=948ee807a1ee817af67f2df94421a202e82325f9
Open pack map
fetchtree:0200289 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=315
git.response.bytes=289
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=86a13f848f75f5d885c75aa02d7eff6ecdc7349d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=289
git.pkt.payload.max=262
git.sideband.band1.bytes=262
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=262
git.pack.trailer=7f5e9a5963305286f535c0b6b2cedf1990871d65
Open pack map
fetch2002.7 KiB315 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=2797
git.response.cache=MISS
git.response.duration_ms=315
git.fetch.wants=917fdd681c954f90037bf9adc715a36e455793ec
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2797
git.pkt.payload.max=2770
git.sideband.band1.bytes=2770
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e28d5bfb0724d3
Request
a8049b37…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format