GitLabgitlab-org
cli
main
1package sync
3import (
4 "bufio"
5 "cmp"
6 "context"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "io"
11 "io/fs"
12 "os"
13 "path/filepath"
14 "slices"
15 "strings"
16 "time"
18 "gitlab.com/gitlab-org/cli/internal/cmdutils"
19 "gitlab.com/gitlab-org/cli/internal/commands/govern/internal/fallbacksync"
20 "gitlab.com/gitlab-org/cli/internal/commands/govern/internal/gaig"
21 "gitlab.com/gitlab-org/cli/internal/dbg"
22)
24// codex syncs Codex sessions, which its Stop hook records and the scheduled
25// job also finds by scanning Codex's rollout files. Their transcript locator
26// is the rollout path.
27type codex struct {
28 executor cmdutils.Executor
29}
31func (codex) name() string { return "codex" }
33func (codex) sessionID() string {
34 id, _ := hookSession("codex")
35 return id
36}
38// current opens the rollout that the Stop hook passed. Subagent threads are
39// skipped, as discover skips them.
40func (codex) current(sessionID string) (transcript, string, error) {
41 _, path := hookSession("codex")
42 meta, err := readCodexSessionMeta(path)
43 if err != nil {
44 return nil, "", err
45 }
46 if meta.ParentThreadID != "" {
47 return nil, "", errors.New("codex subagent threads are not synced")
48 }
49 // Discovery keys the session by this ID, so a different one would sync
50 // it twice.
51 if meta.ID != sessionID {
52 return nil, "", fmt.Errorf("the Codex hook passed session %s for rollout %s", sessionID, meta.ID)
53 }
54 return codexTranscript{sessionID: sessionID, path: path}, path, nil
55}
57func (codex) recorded(sessionID, path string) (transcript, string, error) {
58 return codexTranscript{sessionID: sessionID, path: path}, path, nil
59}
61func (codex) canForget(src transcript, _ time.Time) (bool, error) {
62 t, ok := src.(codexTranscript)
63 if !ok {
64 return false, fmt.Errorf("unexpected Codex transcript type %T", src)
65 }
66 return fileDeleted(t.path)
67}
69func (codex) pausedReason() (string, error) {
70 return discoveryPausedReason("codex")
71}
73// codexSessionsDir is where Codex writes rollouts. CODEX_HOME is followed only
74// when the scheduled job itself has it, which launchd and systemd jobs
75// usually do not.
76func codexSessionsDir() (string, error) {
77 if home := os.Getenv("CODEX_HOME"); home != "" {
78 return filepath.Join(home, "sessions"), nil
79 }
80 home, err := os.UserHomeDir()
81 if err != nil {
82 return "", err
83 }
84 return filepath.Join(home, ".codex", "sessions"), nil
85}
87// codexSessionMeta is the payload of the first line of a rollout.
88type codexSessionMeta struct {
89 ID string `json:"id"`
90 HistoryMode string `json:"history_mode"`
91 Timestamp string `json:"timestamp"`
92 CWD string `json:"cwd"`
93 ParentThreadID string `json:"parent_thread_id"`
94 Git *struct {
95 RepositoryURL string `json:"repository_url"`
96 } `json:"git"`
97}
99func (c codex) discover(ctx context.Context, known func(string) bool) ([]discoveredSession, error) {
100 root, err := codexSessionsDir()
101 if err != nil {
102 return nil, err
103 }
104 var found []discoveredSession
105 // Sessions in the same directory share a remote, so git runs once per
106 // directory in each run.
107 remotes := map[string]string{}
108 err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
109 if errors.Is(err, fs.ErrNotExist) && path == root {
110 return filepath.SkipAll
111 }
112 if err != nil && path == root {
113 return err
114 }
115 if err != nil {
116 // One unreadable directory must not stop discovery of the others.
117 dbg.Debugf("skipping %s: %v", path, err)
118 if d != nil && d.IsDir() {
119 return filepath.SkipDir
120 }
121 return nil
122 }
123 if d.IsDir() || !strings.HasPrefix(d.Name(), "rollout-") || filepath.Ext(d.Name()) != ".jsonl" {
124 return nil
125 }
126 meta, err := readCodexSessionMeta(path)
127 if err != nil {
128 // One unreadable rollout must not stop discovery of the others.
129 dbg.Debugf("skipping %s: %v", path, err)
130 return nil
131 }
132 if meta.ID == "" || known(meta.ID) {
133 return nil
134 }
135 // Subagent threads are not synced in this version.
136 if meta.ParentThreadID != "" {
137 return nil
138 }
139 s := discoveredSession{id: meta.ID, locator: path}
140 if meta.Git != nil && meta.Git.RepositoryURL != "" {
141 s.remote = meta.Git.RepositoryURL
142 } else if meta.CWD != "" {
143 remote, ok := remotes[meta.CWD]
144 if !ok {
145 remote, err = gaig.OriginRemote(ctx, c.executor, meta.CWD)
146 if err != nil {
147 dbg.Debugf("codex session %s: %v", meta.ID, err)
148 }
149 remotes[meta.CWD] = remote
150 }
151 s.remote = remote
152 }
153 found = append(found, s)
154 return nil
155 })
156 return found, err
157}
159func readCodexSessionMeta(path string) (*codexSessionMeta, error) {
160 f, err := os.Open(path)
161 if err != nil {
162 return nil, err
163 }
164 defer f.Close()
165 line, err := bufio.NewReaderSize(f, 1024*1024).ReadBytes('\n')
166 if err != nil && !errors.Is(err, io.EOF) {
167 return nil, err
168 }
169 var first struct {
170 Type string `json:"type"`
171 Payload codexSessionMeta `json:"payload"`
172 }
173 if err := json.Unmarshal(line, &first); err != nil {
174 return nil, err
175 }
176 if first.Type != "session_meta" {
177 return nil, fmt.Errorf("%s does not start with session metadata", path)
178 }
179 return &first.Payload, nil
180}
182// codexTranscript is a Codex rollout. Its cursor is a byte offset.
183type codexTranscript struct {
184 sessionID string
185 path string
186}
188func (t codexTranscript) String() string { return t.path }
190func (t codexTranscript) read(_ context.Context, cursor int64) (*sessionData, int64, error) {
191 if _, err := os.Stat(t.path); errors.Is(err, fs.ErrNotExist) {
192 return nil, cursor, errTranscriptNotFound
193 }
194 // The history mode is only on the first line, and decides where prompts
195 // are recorded.
196 meta, err := readCodexSessionMeta(t.path)
197 if err != nil {
198 return nil, cursor, err
199 }
200 r := codexReader{
201 sessionID: t.sessionID,
202 paginated: meta.HistoryMode == "paginated",
203 data: &sessionData{},
204 goalFound: cursor > 0,
205 calls: map[string]toolCall{},
206 }
207 offset, err := scanJSONL(t.path, cursor, r.parseLine)
208 return r.data, offset, err
209}
211func (t codexTranscript) lastActivity(context.Context) (time.Time, error) {
212 return fileModTime(t.path)
213}
215func (t codexTranscript) checksum(context.Context) (string, error) {
216 return sha256File(t.path)
217}
219// codexToolNames names the response items that are tool calls but carry no
220// name of their own.
221var codexToolNames = map[string]string{
222 "local_shell_call": "shell",
223 "web_search_call": "web_search",
224 "tool_search_call": "tool_search",
225}
227// codexReader collects session data from rollout lines.
228type codexReader struct {
229 sessionID string
230 // paginated rollouts do not record user_message events, so prompts come
231 // from user message items instead.
232 paginated bool
233 data *sessionData
234 goalFound bool
235 calls map[string]toolCall
236 model string
237 // cwd is the folder of the current turn, from the session_meta and
238 // turn_context lines. A read that starts mid-rollout has none until the
239 // next turn, rather than a possibly stale one from session_meta, so the
240 // caller fills in the folder it last saw for those entries.
241 cwd string
242}
244type codexLine struct {
245 Timestamp string `json:"timestamp"`
246 Type string `json:"type"`
247 Payload struct {
248 Type string `json:"type"`
249 ID string `json:"id"`
250 CallID string `json:"call_id"`
251 Name string `json:"name"`
252 Namespace string `json:"namespace"`
253 Role string `json:"role"`
254 Message string `json:"message"`
255 Model string `json:"model"`
256 Arguments json.RawMessage `json:"arguments"`
257 Input string `json:"input"`
258 Action json.RawMessage `json:"action"`
259 Content []struct {
260 Type string `json:"type"`
261 Text string `json:"text"`
262 } `json:"content"`
263 Info *struct {
264 LastTokenUsage struct {
265 InputTokens int `json:"input_tokens"`
266 CachedInputTokens int `json:"cached_input_tokens"`
267 OutputTokens int `json:"output_tokens"`
268 } `json:"last_token_usage"`
269 } `json:"info"`
270 CLIVersion string `json:"cli_version"`
271 CWD string `json:"cwd"`
272 } `json:"payload"`
273}
275func (r *codexReader) parseLine(line []byte, lineOffset int64) {
276 var entry codexLine
277 if json.Unmarshal(line, &entry) != nil {
278 return
279 }
280 ts, err := time.Parse(time.RFC3339Nano, entry.Timestamp)
281 if err != nil {
282 ts = time.Now()
283 }
284 p := entry.Payload
285 lineID := fmt.Sprintf("codex:%s:%d", r.sessionID, lineOffset)
287 switch entry.Type {
288 case "session_meta":
289 r.data.StartedAt = ts
290 r.data.AgentVersion = p.CLIVersion
291 r.setCWD(p.CWD)
292 case "turn_context":
293 r.model = p.Model
294 r.setCWD(p.CWD)
295 case "event_msg":
296 switch p.Type {
297 case "user_message":
298 // user_message events hold the prompt as typed, unlike user
299 // message items, which also carry injected instructions.
300 r.addPrompt(lineID, strings.TrimSpace(p.Message), ts)
301 case "token_count":
302 if p.Info == nil {
303 return
304 }
305 u := p.Info.LastTokenUsage
306 r.data.Responses = append(r.data.Responses, llmResponse{
307 ID: lineID,
308 Model: r.model,
309 Usage: tokenUsage{InputTokens: u.InputTokens, OutputTokens: u.OutputTokens, CacheReadInputTokens: u.CachedInputTokens},
310 Timestamp: ts,
311 CWD: r.cwd,
312 })
313 }
314 case "response_item":
315 r.parseResponseItem(p.Type, entry, lineID, ts)
316 }
317}
319func (r *codexReader) parseResponseItem(itemType string, entry codexLine, lineID string, ts time.Time) {
320 p := entry.Payload
321 switch itemType {
322 case "message":
323 if !r.paginated || p.Role != "user" {
324 return
325 }
326 for _, c := range p.Content {
327 // Injected context is XML-tagged or the AGENTS.md instructions.
328 if c.Type == "input_text" && !strings.HasPrefix(c.Text, "# AGENTS.md") {
329 r.addPrompt(lineID, strings.TrimSpace(c.Text), ts)
330 return
331 }
332 }
333 case "function_call", "custom_tool_call", "local_shell_call", "web_search_call", "tool_search_call":
334 namespace := codexNamespace(p.Namespace)
335 name := codexToolName(namespace, p.Name)
336 if fixed, ok := codexToolNames[itemType]; ok {
337 name = fixed
338 }
339 id := p.CallID
340 if id == "" {
341 id = p.ID
342 }
343 if id == "" {
344 return
345 }
346 input := codexArguments(p.Arguments, p.Input, p.Action)
347 call := toolCall{
348 ID: id, Name: name, Namespace: namespace,
349 Input: input, Timestamp: ts, CWD: r.callCWD(input),
350 }
351 r.calls[id] = call
352 r.data.ToolCalls = append(r.data.ToolCalls, call)
353 case "function_call_output", "custom_tool_call_output":
354 // Codex does not record whether a call succeeded.
355 result := toolResult{CallID: p.CallID, Outcome: "completed", Timestamp: ts, CWD: r.cwd}
356 if call, ok := r.calls[p.CallID]; ok {
357 result.Name = call.Name
358 result.Namespace = call.Namespace
359 result.Duration = ts.Sub(call.Timestamp)
360 result.CWD = call.CWD
361 }
362 r.data.ToolResults = append(r.data.ToolResults, result)
363 }
364}
366func (r *codexReader) addPrompt(id, text string, ts time.Time) {
367 if text == "" || skipPrompt(text) {
368 return
369 }
370 r.data.Prompts = append(r.data.Prompts, prompt{ID: id, Text: text, Timestamp: ts, CWD: r.cwd})
371 if !r.goalFound {
372 r.data.Goal = goalFromText(text)
373 r.goalFound = true
374 }
375}
377func (r *codexReader) setCWD(cwd string) {
378 if cwd != "" {
379 r.cwd = cleanCWD(cwd)
380 }
381}
383// callCWD returns the folder a tool call ran in: a shell call's working
384// directory, which can be relative to the turn's folder, or the turn's folder.
385func (r *codexReader) callCWD(input json.RawMessage) string {
386 var args struct {
387 Workdir string `json:"workdir"`
388 WorkingDirectory string `json:"working_directory"`
389 }
390 // Free-text and other non-object inputs name no working directory.
391 _ = json.Unmarshal(input, &args)
392 dir := cmp.Or(args.Workdir, args.WorkingDirectory)
393 switch {
394 case dir == "":
395 return r.cwd
396 case filepath.IsAbs(dir):
397 return cleanCWD(dir)
398 case r.cwd == "":
399 return ""
400 default:
401 return filepath.Join(r.cwd, dir)
402 }
403}
405// codexNamespace drops the trailing `__` separator older rollouts add, so both
406// rollout shapes send the same namespace. Only the separator goes, so a server
407// name ending in `_` keeps it. A namespace of only underscores becomes empty.
408func codexNamespace(namespace string) string {
409 if strings.Trim(namespace, "_") == "" {
410 return ""
411 }
412 return strings.TrimSuffix(namespace, "__")
413}
415// codexToolName qualifies a tool name with its namespace the way Codex names
416// MCP tools for the model, `mcp__<server>__<tool>`. Built-in tools have no
417// namespace and keep their bare name. The namespace must already be
418// normalized by codexNamespace.
419func codexToolName(namespace, name string) string {
420 if namespace == "" {
421 return name
422 }
423 return namespace + "__" + name
424}
426// codexArguments returns a tool call's arguments as JSON: function call
427// arguments are a string that holds JSON, tool search arguments are a JSON
428// value, custom tool input is free text, and shell and web search calls carry
429// an action object.
430func codexArguments(arguments json.RawMessage, input string, action json.RawMessage) json.RawMessage {
431 var text string
432 isString := json.Unmarshal(arguments, &text) == nil
433 switch {
434 case isString && text != "" && json.Valid([]byte(text)):
435 return json.RawMessage(text)
436 case isString && text != "":
437 return marshalString(text)
438 case !isString && len(arguments) > 0 && string(arguments) != "null":
439 return arguments
440 case input != "":
441 encoded, err := json.Marshal(map[string]string{"input": input}) //nolint:forbidigo // not output
442 if err != nil {
443 return nil
444 }
445 return encoded
446 default:
447 return action
448 }
449}
451func marshalString(s string) json.RawMessage {
452 encoded, err := json.Marshal(s) //nolint:forbidigo // not output
453 if err != nil {
454 return nil
455 }
456 return encoded
457}
459// discoveryPausedReason pauses an agent that setup no longer enables.
460func discoveryPausedReason(agent string) (string, error) {
461 enabled, err := fallbacksync.DiscoveredAgents()
462 if err != nil {
463 return "", err
464 }
465 if slices.Contains(enabled, agent) {
466 return "", nil
467 }
468 return fmt.Sprintf("%s is not enabled; run 'glab minder setup --agents %s' to enable it", agent, agent), nil
469}
Diagnostics9 git calls · 242 ms
CallStatusSizeTime0242 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchtree:02001.8 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=1857 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1857 git.pkt.payload.max=1830 git.sideband.band1.bytes=1830 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1830 git.pack.trailer=fb03183b7909876778f740d196d7516811003517Open pack map
fetchtree:0200261 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=261 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=3bb9946301aa8c0dc8d202b83bc9f84bf0400f6d git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=261 git.pkt.payload.max=234 git.sideband.band1.bytes=234 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=234 git.pack.trailer=e0a52f164f79506da6b0dcc275ed62911f5383bfOpen pack map
fetchtree:0200139 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=139 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=f91da6c620e9fe39086f689d1ae91b87356cda50 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=139 git.pkt.payload.max=112 git.sideband.band1.bytes=112 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=112 git.pack.trailer=a35549813b2a01d473947912b5773e993ce7c2e2Open pack map
fetchtree:0200638 B · hit3 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=638 git.response.cache=HIT git.response.duration_ms=3 git.fetch.filter=tree:0 git.fetch.wants=02089b627a9895becfd23127f25e46df629ce9b2 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=638 git.pkt.payload.max=611 git.sideband.band1.bytes=611 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=611 git.pack.trailer=f2a6246e346c2fa34e54ecb04cb1dd1c524f65bcOpen pack map
fetch2004.7 KiB228 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=228 git.response.bytes=4816 git.response.cache=MISS git.response.duration_ms=228 git.fetch.wants=b761a16cc8d0613f4d5476ff15fc0346a6ac9d1b git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=4816 git.pkt.payload.max=4789 git.sideband.band1.bytes=4789 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e2d1302857cf3a
- Request
- dd2ffed1…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format