GitLabgitlab-org

cli

main
ply9 git calls317 ms100 KiB7 cachedprotocol v2
284 lines7.8 KiBRaw
1package catalog
3import (
4 "fmt"
5 "net/http"
6 "net/url"
7 "os"
8 "path/filepath"
9 "sort"
11 "github.com/MakeNowJust/heredoc/v2"
12 "github.com/spf13/cobra"
13 "go.yaml.in/yaml/v3"
15 gitlab "gitlab.com/gitlab-org/api/client-go/v3"
17 "gitlab.com/gitlab-org/cli/internal/cmdutils"
18 "gitlab.com/gitlab-org/cli/internal/glrepo"
19 "gitlab.com/gitlab-org/cli/internal/iostreams"
20 "gitlab.com/gitlab-org/cli/internal/mcpannotations"
23const (
24 publishToCatalogApiPath = "projects/%s/catalog/publish"
25 templatesDirName = "templates"
26 templateFileExt = ".yml"
27 templateFileName = "template.yml"
30type publishToCatalogRequest struct {
31 Version string `json:"version"`
32 Metadata map[string]any `json:"metadata"`
35type publishToCatalogResponse struct {
36 CatalogUrl string `json:"catalog_url"`
39type options struct {
40 tagName string
42 gitlabClient func() (*gitlab.Client, error)
43 baseRepo func() (glrepo.Interface, error)
44 io *iostreams.IOStreams
47func NewCmdPublishCatalog(f cmdutils.Factory) *cobra.Command {
48 opts := &options{
49 io: f.IO(),
50 gitlabClient: f.GitLabClient,
51 baseRepo: f.BaseRepo,
52 }
53 publishCatalogCmd := &cobra.Command{
54 Use: "catalog <tag-name>",
55 Short: `[EXPERIMENTAL] Publishes CI/CD components to the catalog.`,
56 Long: heredoc.Docf(`[EXPERIMENTAL] Publishes CI/CD components in the project to the CI/CD catalog using the provided tag name.
58 Requires the feature flag %[1]sci_release_cli_catalog_publish_option%[1]s to be enabled
59 for this project in your GitLab instance.
61 Requires the same user as the release author.
63 - It retrieves components from the current repository by searching for
64 %[1]syml%[1]s files within the "templates" directory and its subdirectories.
65 - It fails if the feature flag %[1]sci_release_cli_catalog_publish_option%[1]s
66 is not enabled for this project in your GitLab instance.
68 Components can be defined:
70 - In single files ending in %[1]s.yml%[1]s for each component, like %[1]stemplates/secret-detection.yml%[1]s.
71 - In subdirectories containing %[1]stemplate.yml%[1]s files as entry points,
72 for components that bundle together multiple related files. For example,
73 %[1]stemplates/secret-detection/template.yml%[1]s.
74 `, "`"),
75 Example: heredoc.Doc(`
76 glab repo publish catalog v1.2.3`),
77 Args: cobra.ExactArgs(1),
78 Annotations: map[string]string{
79 mcpannotations.Destructive: "true",
80 },
81 RunE: func(cmd *cobra.Command, args []string) error {
82 opts.complete(args)
84 return opts.run()
85 },
86 }
88 return publishCatalogCmd
91func (o *options) complete(args []string) {
92 o.tagName = args[0]
95func (o *options) run() error {
96 repo, err := o.baseRepo()
97 if err != nil {
98 return err
99 }
101 client, err := o.gitlabClient()
102 if err != nil {
103 return err
104 }
106 _, _, err = client.Tags.GetTag(repo.FullName(), o.tagName)
107 if err != nil {
108 return &cmdutils.FlagError{Err: fmt.Errorf("invalid tag %s", o.tagName)}
109 }
111 return Publish(o.io, client, repo.FullName(), o.tagName)
114func Publish(io *iostreams.IOStreams, client *gitlab.Client, repoFullName string, tagName string) error {
115 color := io.Color()
117 io.LogInfof("%s Publishing release %s=%s to the GitLab CI/CD catalog for %s=%s...\n",
118 color.ProgressIcon(),
119 color.Blue("tag"), tagName,
120 color.Blue("repo"), repoFullName)
122 body, err := publishToCatalogRequestBody(tagName)
123 if err != nil {
124 return cmdutils.WrapError(err, "failed to create a request body.")
125 }
127 path := fmt.Sprintf(publishToCatalogApiPath, url.PathEscape(repoFullName))
128 request, err := client.NewRequest(http.MethodPost, path, body, nil)
129 if err != nil {
130 return cmdutils.WrapError(err, "failed to create a request.")
131 }
133 var response publishToCatalogResponse
134 _, err = client.Do(request, &response)
135 if err != nil {
136 return err
137 }
139 io.LogInfof("%s Release published: %s=%s\n", color.GreenCheck(),
140 color.Blue("url"), response.CatalogUrl)
142 return nil
145func publishToCatalogRequestBody(version string) (*publishToCatalogRequest, error) {
146 baseDir, err := os.Getwd()
147 if err != nil {
148 return nil, cmdutils.WrapError(err, "failed to get working directory")
149 }
151 components, err := fetchTemplates(baseDir)
152 if err != nil {
153 return nil, cmdutils.WrapError(err, "failed to fetch components")
154 }
156 metadata := make(map[string]any)
157 componentsData := make([]map[string]any, 0, len(components))
159 for name, path := range components {
160 spec, err := extractSpec(path)
161 if err != nil {
162 return nil, cmdutils.WrapError(err, "failed to extract spec")
163 }
165 componentsData = append(componentsData, map[string]any{
166 "name": name,
167 "spec": spec,
168 "component_type": "template",
169 })
170 }
172 sort.Slice(componentsData, func(i, j int) bool {
173 // `name` is always set to a string literal a few lines above.
174 ni, _ := componentsData[i]["name"].(string)
175 nj, _ := componentsData[j]["name"].(string)
176 return ni < nj
177 })
179 metadata["components"] = componentsData
181 return &publishToCatalogRequest{
182 Version: version,
183 Metadata: metadata,
184 }, nil
187// fetchTemplates returns a map of component names to their paths.
188// The component name is either the name of the file without the extension in the "templates" directory of the project
189// or the name of the directory containing a "template.yml" file in the "templates" directory.
190// More information: https://docs.gitlab.com/ci/components/#directory-structure
191func fetchTemplates(baseDir string) (map[string]string, error) {
192 templates := make(map[string]string)
194 paths, err := fetchTemplatePaths(baseDir)
195 if err != nil {
196 return nil, cmdutils.WrapError(err, "failed to fetch template paths")
197 }
199 for _, path := range paths {
200 componentName, err := extractComponentName(baseDir, path)
201 if err != nil {
202 return nil, cmdutils.WrapError(err, "failed to extract component name")
203 }
205 if componentName != "" {
206 templates[componentName] = path
207 }
208 }
210 return templates, nil
213// fetchTemplatePaths returns a list of the possible component paths to the YAML files in the "templates" directory.
214func fetchTemplatePaths(baseDir string) ([]string, error) {
215 templatesDir := filepath.Join(baseDir, templatesDirName)
217 var yamlFiles []string
219 err := filepath.WalkDir(templatesDir, func(path string, d os.DirEntry, err error) error {
220 if err != nil {
221 return cmdutils.WrapError(err, "failed to walk directory")
222 }
224 if filepath.Ext(d.Name()) == templateFileExt {
225 yamlFiles = append(yamlFiles, path)
226 }
228 return nil
229 })
230 if err != nil {
231 return nil, err
232 }
234 return yamlFiles, nil
237// extractComponentName returns the valid component name from the path if it is a valid component path.
238// valid component paths:
239// 1. All YAML files in the "templates" directory.
240// 2. All "template.yml" files in the subdirectories of the "templates" directory.
241func extractComponentName(baseDir string, path string) (string, error) {
242 relativePath, err := filepath.Rel(baseDir, path)
243 if err != nil {
244 return "", err
245 }
247 dirname := filepath.Dir(relativePath)
248 fileExt := filepath.Ext(relativePath)
249 filename := filepath.Base(relativePath)
250 filenameWithoutExt := filename[:len(filename)-len(fileExt)]
252 // All YAML files in the "templates" directory.
253 if dirname == templatesDirName {
254 return filenameWithoutExt, nil
255 }
257 // All "template.yml" files in the subdirectories of the "templates" directory.
258 if filename == templateFileName {
259 return filepath.Base(dirname), nil
260 } else {
261 return "", nil
262 }
265type specDef struct {
266 Spec map[string]any `yaml:"spec"`
269// extractSpec returns the spec from the component file.
270func extractSpec(componentPath string) (map[string]any, error) {
271 content, err := os.ReadFile(componentPath)
272 if err != nil {
273 return nil, cmdutils.WrapError(err, "failed to read file")
274 }
276 var spec specDef
278 err = yaml.Unmarshal(content, &spec)
279 if err != nil {
280 return nil, cmdutils.WrapError(err, "failed to unmarshal YAML")
281 }
283 return spec.Spec, nil
Diagnostics9 git calls · 317 ms
CallStatusSizeTime0317 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:02001.8 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=1857
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1857
git.pkt.payload.max=1830
git.sideband.band1.bytes=1830
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1830
git.pack.trailer=fb03183b7909876778f740d196d7516811003517
Open pack map
fetchtree:0200614 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=614
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=3a89d0cfc8e81880c982180388e42e561a0b7eee
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=614
git.pkt.payload.max=587
git.sideband.band1.bytes=587
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=587
git.pack.trailer=eb8250f8e8340ff2fafc22c5aba90db34fd906a9
Open pack map
fetchtree:0200142 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=142
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=af7f92870be97ec283a67c3dc90186c455c369ae
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=142
git.pkt.payload.max=115
git.sideband.band1.bytes=115
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=115
git.pack.trailer=93031c1d64f40c6c49227c17486b6689427c106d
Open pack map
fetchtree:0200172 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=172
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=4dea86ad6397a41b9cbead34fc69a4c4244f25e7
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=172
git.pkt.payload.max=145
git.sideband.band1.bytes=145
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=145
git.pack.trailer=2bd83ce989fb929895280a53fa461cd216c8ad4d
Open pack map
fetch2002.6 KiB302 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=302
git.response.bytes=2645
git.response.cache=MISS
git.response.duration_ms=302
git.fetch.wants=63afc86bbfbe65effec525cead93800589321b4d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2645
git.pkt.payload.max=2618
git.sideband.band1.bytes=2618
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5ea625b2c09ecd4
Request
10230b3b…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format