GitLabgitlab-org
cli
main
1package remote
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "regexp"
11 "slices"
12 "strconv"
13 "strings"
14 "time"
16 "gitlab.com/gitlab-org/cli/internal/commands/skills/skill"
17)
19// gitlabHost is hardcoded — curated remote skills always live on
20// gitlab.com regardless of where the user's `glab` is configured.
21const (
22 gitlabHost = "https://gitlab.com"
23 apiV4 = gitlabHost + "/api/v4"
24 defaultTimeout = 30 * time.Second
25 // treePageSize matches GitLab's max per_page for repository listings.
26 treePageSize = 100
27)
29// fetcher pulls a skill's full file tree from gitlab.com via the
30// public Repository APIs (no auth). It is constructed per-Get so the
31// HTTP client can be swapped in tests.
32type fetcher struct {
33 client *http.Client
34 api string
35 host string
36}
38func newFetcher(client *http.Client) *fetcher {
39 if client == nil {
40 client = http.DefaultClient
41 }
42 return &fetcher{
43 client: client,
44 api: apiV4,
45 host: gitlabHost,
46 }
47}
49// fetch resolves the entry's ref (handling the `latest` shortcut) and
50// downloads every blob under entry.Path, returning a Skill keyed by
51// paths relative to entry.Path.
52func (f *fetcher) fetch(e Entry) (skill.Skill, error) {
53 ref, err := f.resolveRef(e.Project, e.Ref)
54 if err != nil {
55 return skill.Skill{}, err
56 }
58 tree, err := f.listTree(e.Project, e.Path, ref)
59 if err != nil {
60 return skill.Skill{}, fmt.Errorf("listing %s@%s:%s: %w", e.Project, ref, e.Path, err)
61 }
63 files := map[string][]byte{}
64 for _, te := range tree {
65 if te.Type != "blob" {
66 continue
67 }
68 body, err := f.getRaw(e.Project, ref, te.Path)
69 if err != nil {
70 return skill.Skill{}, fmt.Errorf("fetching %s: %w", te.Path, err)
71 }
72 rel, err := relPath(e.Path, te.Path)
73 if err != nil {
74 return skill.Skill{}, err
75 }
76 files[rel] = body
77 }
79 if _, ok := files[skill.FileName]; !ok {
80 return skill.Skill{}, fmt.Errorf("remote skill %q at %s@%s:%s has no %s", e.Name, e.Project, ref, e.Path, skill.FileName)
81 }
83 renamed, err := setFrontmatterName(files[skill.FileName], e.Name)
84 if err != nil {
85 return skill.Skill{}, fmt.Errorf("remote skill %q at %s@%s:%s: %w", e.Name, e.Project, ref, e.Path, err)
86 }
87 files[skill.FileName] = renamed
89 return skill.Skill{
90 Name: e.Name,
91 Description: e.Description,
92 Source: skill.SourceRemote,
93 Files: files,
94 }, nil
95}
97var frontmatterNameLine = regexp.MustCompile(`(?m)^name:[^\r\n]*`)
99// setFrontmatterName rewrites the top-level `name:` of a SKILL.md
100// frontmatter so it matches the registry name, which is also the install
101// directory. The agent skills spec requires the two to be equal, and an
102// upstream skill may legitimately publish under a different name.
103func setFrontmatterName(content []byte, name string) ([]byte, error) {
104 parts, err := skill.SplitFrontmatter(content)
105 if err != nil {
106 return nil, fmt.Errorf("%s: %w", skill.FileName, err)
107 }
108 loc := frontmatterNameLine.FindIndex(parts.Block)
109 if loc == nil {
110 return nil, fmt.Errorf("%s frontmatter is missing 'name'", skill.FileName)
111 }
112 parts.Block = slices.Concat(parts.Block[:loc[0]], []byte("name: "+name), parts.Block[loc[1]:])
114 rewritten := parts.Join()
115 fm, err := skill.ParseFrontmatter(rewritten)
116 if err != nil {
117 return nil, fmt.Errorf("%s: %w", skill.FileName, err)
118 }
119 if fm.Name != name {
120 return nil, fmt.Errorf("%s frontmatter name is %q after rewrite, want %q", skill.FileName, fm.Name, name)
121 }
122 return rewritten, nil
123}
125// resolveRef expands the `latest` shortcut to the project's default
126// branch. Other ref strings (tags, SHAs, branch names) pass through.
127func (f *fetcher) resolveRef(project, ref string) (string, error) {
128 if ref != "latest" {
129 return ref, nil
130 }
131 u := f.api + "/projects/" + url.PathEscape(project)
132 body, err := f.getJSON(u)
133 if err != nil {
134 return "", fmt.Errorf("resolving 'latest' for %s: %w", project, err)
135 }
136 var info struct {
137 DefaultBranch string `json:"default_branch"`
138 }
139 if err := json.Unmarshal(body, &info); err != nil {
140 return "", fmt.Errorf("parsing project metadata for %s: %w", project, err)
141 }
142 if info.DefaultBranch == "" {
143 return "", fmt.Errorf("project %s has no default_branch", project)
144 }
145 return info.DefaultBranch, nil
146}
148type treeEntry struct {
149 Path string `json:"path"`
150 Type string `json:"type"` // "blob" | "tree" | "commit"
151}
153// listTree pages through /repository/tree until exhausted.
154func (f *fetcher) listTree(project, path, ref string) ([]treeEntry, error) {
155 var all []treeEntry
156 base := f.api + "/projects/" + url.PathEscape(project) + "/repository/tree"
158 for page := 1; ; page++ {
159 q := url.Values{}
160 q.Set("path", path)
161 q.Set("ref", ref)
162 q.Set("recursive", "true")
163 q.Set("per_page", strconv.Itoa(treePageSize))
164 q.Set("page", strconv.Itoa(page))
166 body, hdr, err := f.getJSONWithHeaders(base + "?" + q.Encode())
167 if err != nil {
168 return nil, err
169 }
171 var batch []treeEntry
172 if err := json.Unmarshal(body, &batch); err != nil {
173 return nil, fmt.Errorf("parsing tree response: %w", err)
174 }
175 all = append(all, batch...)
177 next := hdr.Get("X-Next-Page")
178 if next == "" || next == "0" {
179 break
180 }
181 }
182 return all, nil
183}
185// getRaw fetches a single file's raw bytes via the non-API raw URL,
186// which avoids URL-encoding the path with %2F separators.
187func (f *fetcher) getRaw(project, ref, path string) ([]byte, error) {
188 u := f.host + "/" + project + "/-/raw/" + ref + "/" + path
189 body, _, err := f.httpGet(u, nil)
190 return body, err
191}
193func (f *fetcher) getJSON(u string) ([]byte, error) {
194 body, _, err := f.httpGet(u, http.Header{"Accept": []string{"application/json"}})
195 return body, err
196}
198func (f *fetcher) getJSONWithHeaders(u string) ([]byte, http.Header, error) {
199 return f.httpGet(u, http.Header{"Accept": []string{"application/json"}})
200}
202// httpGet issues a GET against u with the given headers, reads the body,
203// and returns the bytes + response headers. The per-request timeout is
204// applied via context only when neither the client nor the parent
205// context already carries a deadline; the cancel scope spans the body
206// read so a default-timed fetch isn't cut off mid-stream.
207func (f *fetcher) httpGet(u string, headers http.Header) ([]byte, http.Header, error) {
208 ctx := context.Background()
209 if _, ok := ctx.Deadline(); !ok && f.client.Timeout == 0 {
210 var cancel context.CancelFunc
211 ctx, cancel = context.WithTimeout(ctx, defaultTimeout)
212 defer cancel()
213 }
215 req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
216 if err != nil {
217 return nil, nil, err
218 }
219 for k, vs := range headers {
220 for _, v := range vs {
221 req.Header.Add(k, v)
222 }
223 }
225 resp, err := f.client.Do(req)
226 if err != nil {
227 return nil, nil, err
228 }
229 defer resp.Body.Close()
231 if resp.StatusCode != http.StatusOK {
232 return nil, nil, fmt.Errorf("GET %s: %s", u, resp.Status)
233 }
235 body, err := io.ReadAll(resp.Body)
236 if err != nil {
237 return nil, nil, err
238 }
239 return body, resp.Header, nil
240}
242func relPath(root, full string) (string, error) {
243 if full == root {
244 return "", fmt.Errorf("tree entry %q equals skill root", full)
245 }
246 if !strings.HasPrefix(full, root+"/") {
247 return "", fmt.Errorf("tree entry %q is not under skill root %q", full, root)
248 }
249 return full[len(root)+1:], nil
250}
Diagnostics8 git calls · 49 ms
CallStatusSizeTime049 ms
discover200185 B3 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=185 git.server.agent=git/2.56.g1891de4-Linux git.server.object_format=sha1 git.server.caps=agent ls-refs fetch server-option object-format git.server.fetch=shallow wait-for-done filter git.protocol.version=2 git.pkt.frames=9 git.pkt.data=7 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=185 git.pkt.payload.max=35
ls-refs20058 KiB · hit3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=59849 git.response.cache=HIT git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=731 git.pkt.data=730 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=59849 git.pkt.payload.max=143 git.ls_refs.count=730 git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=4273 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=combine:blob:none+tree:2 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=1 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=7 git.pkt.data=5 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=4273 git.pkt.payload.max=4173 git.sideband.band1.bytes=4173 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=11 git.pack.bytes=4173 git.fetch.wide.objects=11 git.fetch.wide.extra_bytes=3930 git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=1 git.response.bytes=32926 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9 git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=22 git.pkt.data=20 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=32926 git.pkt.payload.max=32046 git.sideband.band1.bytes=32046 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=80 git.pack.bytes=32046 git.pack.ofs_delta=36 git.pack.ref_delta=0 git.pack.chain.max=2 git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91 git.pack.full=44Open pack map
fetchtree:02001.8 KiB · hit1 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=2 git.response.bytes=1857 git.response.cache=HIT git.response.duration_ms=1 git.fetch.filter=tree:0 git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=1857 git.pkt.payload.max=1830 git.sideband.band1.bytes=1830 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=1830 git.pack.trailer=fb03183b7909876778f740d196d7516811003517Open pack map
fetchtree:0200370 B · hit2 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=4 git.response.bytes=370 git.response.cache=HIT git.response.duration_ms=2 git.fetch.filter=tree:0 git.fetch.wants=290ae7a62f76e23cbbd60697570f19b90ddcece6 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=370 git.pkt.payload.max=343 git.sideband.band1.bytes=343 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=343 git.pack.trailer=0795a26d2b25022c69da2ef2e68cfb92a77ef4bbOpen pack map
fetchtree:0200205 B · hit4 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=37 git.response.bytes=205 git.response.cache=HIT git.response.duration_ms=4 git.fetch.filter=tree:0 git.fetch.wants=81dc066a953916d0f0a002cd7235dfc47cfbe9ea git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=205 git.pkt.payload.max=178 git.sideband.band1.bytes=178 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=1 git.pack.bytes=178 git.pack.trailer=a6b69d21421b8ef5347f95fc10c7bb5be79155a9Open pack map
fetch2002.7 KiB · hit37 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2796 git.response.cache=HIT git.response.duration_ms=37 git.fetch.wants=3d7118171c9cc00b4a6f3004e2b5851328567847 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=4 git.pkt.data=3 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2796 git.pkt.payload.max=2769 git.sideband.band1.bytes=2769 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://gitlab.com/gitlab-org/cli
- Ref
- refs/heads/main
- Commit
- 0dd7574dce9e385c5c30540e6c15657adc6bdce9
- cf-ray
- a5e2d11a8f63cf3a
- Request
- affc86fb…
- Protocol
- v2 · sha1
- Agent
- git/2.56.g1891de4-Linux
- Filter
- advertised
- Fetch
- shallow wait-for-done filter
- Capabilities
- agent ls-refs fetch server-option object-format