GitLabgitlab-org

cli

main
ply9 git calls232 ms100 KiB6 cachedprotocol v2
287 lines8.3 KiBRaw
1package upload
3import (
4 "context"
5 "fmt"
6 "io"
7 "net/url"
9 "golang.org/x/sync/errgroup"
11 gitlab "gitlab.com/gitlab-org/api/client-go/v3"
13 "gitlab.com/gitlab-org/cli/internal/iostreams"
16// DefaultReleasePackageName contains the name for the release asset package
17const DefaultReleasePackageName = "release-assets"
19const uploadConcurrency = 4
21// ConflictDirectAssetPathError is returned when both direct_asset_path and the deprecated filepath as specified for an asset link.
22type ConflictDirectAssetPathError struct {
23 assetLinkName *string
26func (e *ConflictDirectAssetPathError) Error() string {
27 var name string
28 if e.assetLinkName != nil {
29 name = *e.assetLinkName
30 } else {
31 name = "(without a name)"
32 }
33 return fmt.Sprintf("asset link %s contains both `direct_asset_path` and `filepath` (deprecated) fields. Remove the deprecated `filepath` field.", name)
36type ReleaseAsset struct {
37 Name *string `json:"name,omitempty"`
38 URL *string `json:"url,omitempty"`
39 // Deprecated FilePath use DirectAssetPath instead.
40 FilePath *string `json:"filepath,omitempty"`
41 DirectAssetPath *string `json:"direct_asset_path,omitempty"`
42 LinkType *gitlab.LinkTypeValue `json:"link_type,omitempty"`
45type ReleaseFile struct {
46 Open func() (io.ReadCloser, error)
47 Name string
48 Label string
49 Path string
50 Type *gitlab.LinkTypeValue
53func CreateLink(ctx context.Context, c *gitlab.Client, projectID, tagName string, asset *ReleaseAsset) (*gitlab.ReleaseLink, bool /* aliased deprecated filepath */, error) {
54 aliased, err := aliasFilePathToDirectAssetPath(asset)
55 if err != nil {
56 return nil, false, err
57 }
59 releaseLink, _, err := c.ReleaseLinks.CreateReleaseLink(projectID, tagName, &gitlab.CreateReleaseLinkOptions{
60 Name: asset.Name,
61 URL: asset.URL,
62 DirectAssetPath: asset.DirectAssetPath,
63 LinkType: asset.LinkType,
64 }, gitlab.WithContext(ctx))
65 if err != nil {
66 return nil, false, err
67 }
69 return releaseLink, aliased, nil
72type Context struct {
73 Client *gitlab.Client
74 IO *iostreams.IOStreams
75 AssetFiles []*ReleaseFile
76 AssetsLinks []*ReleaseAsset
79// UploadFiles uploads the asset files concurrently and then creates their
80// release links in input order, so the release lists assets deterministically.
81func (c *Context) UploadFiles(ctx context.Context, projectID, tagName string, packageName string, usePackageRegistry bool) error {
82 if c.AssetFiles == nil {
83 return nil
84 }
85 releaseAssets, err := c.UploadAssetFiles(ctx, projectID, tagName, packageName, usePackageRegistry)
86 if err != nil {
87 return err
88 }
90 for _, releaseAsset := range releaseAssets {
91 if _, _, err := CreateLink(ctx, c.Client, projectID, tagName, releaseAsset); err != nil {
92 return err
93 }
94 }
95 c.AssetFiles = nil
97 return nil
100// UploadAssetFiles does not link the files. The release may not exist yet.
101func (c *Context) UploadAssetFiles(ctx context.Context, projectID, tagName string, packageName string, usePackageRegistry bool) ([]*ReleaseAsset, error) {
102 color := c.IO.Color()
103 for _, file := range c.AssetFiles {
104 c.IO.LogInfof("%s Uploading to release\t%s=%s %s=%s\n",
105 color.ProgressIcon(), color.Blue("file"), file.Path,
106 color.Blue("name"), file.Name)
107 }
109 g, groupCtx := errgroup.WithContext(ctx)
110 g.SetLimit(uploadConcurrency)
111 releaseAssets := make([]*ReleaseAsset, len(c.AssetFiles))
112 for i, file := range c.AssetFiles {
113 g.Go(func() error {
114 if err := groupCtx.Err(); err != nil {
115 return err
116 }
117 var err error
118 if usePackageRegistry {
119 releaseAssets[i], err = c.uploadAsGenericPackage(groupCtx, projectID, tagName, packageName, file)
120 } else {
121 releaseAssets[i], err = c.uploadAsProjectMarkdownFile(groupCtx, projectID, file)
122 }
123 if err != nil {
124 return fmt.Errorf("uploading %s: %w", file.Path, err)
125 }
126 return nil
127 })
128 }
129 if err := g.Wait(); err != nil {
130 return nil, err
131 }
133 return releaseAssets, nil
136func (c *Context) uploadAsGenericPackage(ctx context.Context, projectID, tagName string, packageName string, file *ReleaseFile) (*ReleaseAsset, error) {
137 r, err := file.Open()
138 if err != nil {
139 return nil, err
140 }
141 defer r.Close()
143 _, _, err = c.Client.GenericPackages.PublishPackageFile(projectID, packageName, tagName, file.Name, r, nil, gitlab.WithContext(ctx))
144 if err != nil {
145 return nil, err
146 }
148 assetPath, err := c.Client.GenericPackages.FormatPackageURL(projectID, packageName, tagName, file.Name)
149 if err != nil {
150 return nil, err
151 }
152 assetURL := c.Client.BaseURL().JoinPath(assetPath)
153 return &ReleaseAsset{
154 Name: new(file.Label),
155 URL: new(assetURL.String()),
156 DirectAssetPath: new("/" + url.PathEscape(file.Name)),
157 LinkType: file.Type,
158 }, nil
161func (c *Context) uploadAsProjectMarkdownFile(ctx context.Context, projectID string, file *ReleaseFile) (*ReleaseAsset, error) {
162 r, err := file.Open()
163 if err != nil {
164 return nil, err
165 }
166 defer r.Close()
168 projectFile, _, err := c.Client.ProjectMarkdownUploads.UploadProjectMarkdown(
169 projectID,
170 r,
171 file.Name,
172 gitlab.WithContext(ctx),
173 )
174 if err != nil {
175 return nil, err
176 }
178 // projectFile.FullPath contains the absolute path of the file within the
179 // context of the client's assetURL.
180 //
181 // It has a structure like so:
182 // FullPath: /-/project/[project_id]/uploads/[hash]/[file]
183 // Or on older GitLab prior to 17:
184 // FullPath: /[namespace]/[project]/uploads/[hash]/[file]
185 //
186 // Thus, we append it to the base URL to get a usable link.
187 //
188 // See for context https://docs.gitlab.com/api/project_markdown_uploads/
189 assetURL := c.Client.BaseURL()
190 assetURL.Path = projectFile.FullPath
192 return &ReleaseAsset{
193 Name: new(file.Label),
194 URL: new(assetURL.String()),
195 DirectAssetPath: new("/" + url.PathEscape(file.Name)),
196 LinkType: file.Type,
197 }, nil
200func (c *Context) CreateReleaseAssetLinks(ctx context.Context, projectID string, tagName string) error {
201 if c.AssetsLinks == nil {
202 return nil
203 }
204 color := c.IO.Color()
205 for _, asset := range c.AssetsLinks {
206 releaseLink, aliased, err := CreateLink(ctx, c.Client, projectID, tagName, asset)
207 if err != nil {
208 return err
209 }
210 c.IO.LogInfof("%s Added release asset\t%s=%s %s=%s\n",
211 color.GreenCheck(), color.Blue("name"), *asset.Name,
212 color.Blue("url"), releaseLink.DirectAssetURL)
214 if aliased {
215 c.IO.LogInfof("\t%s Aliased deprecated `filepath` field to `direct_asset_path`. Replace `filepath` with `direct_asset_path`\t%s=%s\n",
216 color.WarnIcon(),
217 color.Blue("name"), *asset.Name)
218 }
220 }
221 c.AssetsLinks = nil
223 return nil
226// AliasFilePaths returns the names of the assets whose deprecated filepath it
227// moved to direct_asset_path.
228func AliasFilePaths(assets []*ReleaseAsset) ([]string, error) {
229 var aliasedNames []string
230 for _, asset := range assets {
231 aliased, err := aliasFilePathToDirectAssetPath(asset)
232 if err != nil {
233 return nil, err
234 }
235 if aliased {
236 name := "(without a name)"
237 if asset.Name != nil {
238 name = *asset.Name
239 }
240 aliasedNames = append(aliasedNames, name)
241 }
242 }
244 return aliasedNames, nil
247// ToReleaseAssetsOptions ignores the deprecated filepath. Call AliasFilePaths
248// first.
249func ToReleaseAssetsOptions(assets []*ReleaseAsset) *gitlab.ReleaseAssetsOptions {
250 if len(assets) == 0 {
251 return nil
252 }
254 links := make([]*gitlab.ReleaseAssetLinkOptions, 0, len(assets))
255 for _, asset := range assets {
256 links = append(links, &gitlab.ReleaseAssetLinkOptions{
257 Name: asset.Name,
258 URL: asset.URL,
259 DirectAssetPath: asset.DirectAssetPath,
260 LinkType: asset.LinkType,
261 })
262 }
264 return &gitlab.ReleaseAssetsOptions{Links: links}
267// aliasFilePathToDirectAssetPath ensures that the Asset Link uses the direct_asset_path and not the filepath.
268// The filepath is deprecated and will be fully removed in GitLab 17.0.
269// See https://gitlab.com/groups/gitlab-org/-/work_items/9661
270func aliasFilePathToDirectAssetPath(asset *ReleaseAsset) (bool /* aliased */, error) {
271 if asset.FilePath == nil || *asset.FilePath == "" {
272 // There is no deprecated filepath set, so we are good.
273 return false, nil
274 }
276 if asset.DirectAssetPath != nil && *asset.DirectAssetPath != "" {
277 // Both, filepath and direct_asset_path are set, so we have a conflict
278 return false, &ConflictDirectAssetPathError{asset.Name}
279 }
281 // We use the set filepath as direct_asset_path
282 // and clear the filepath to net send it via API.
283 asset.DirectAssetPath = asset.FilePath
284 asset.FilePath = nil
286 return true, nil
Diagnostics9 git calls · 232 ms
CallStatusSizeTime0232 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB8 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=UPDATING
git.response.duration_ms=8
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:02001.8 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=1857
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1857
git.pkt.payload.max=1830
git.sideband.band1.bytes=1830
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1830
git.pack.trailer=fb03183b7909876778f740d196d7516811003517
Open pack map
fetchtree:0200350 B · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=350
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=b2a837e646ba305993f35203fafafb109e57a7ca
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=350
git.pkt.payload.max=323
git.sideband.band1.bytes=323
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=323
git.pack.trailer=1232131042f2ba4c9c4dbd99382667a1f0849e15
Open pack map
fetchtree:0200145 B · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=145
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=tree:0
git.fetch.wants=fd939ecfee6b4c8fd9edfe7831322b170467757f
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=145
git.pkt.payload.max=118
git.sideband.band1.bytes=118
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=118
git.pack.trailer=659e45f50374d89b1e4735a4852bfdc2e0e55abc
Open pack map
fetchtree:0200139 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=139
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=6e44dc08508d39aa42540b4afc6fdc5734da04c4
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=139
git.pkt.payload.max=112
git.sideband.band1.bytes=112
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=112
git.pack.trailer=f3003d1df2e47d08c612d559da907518ff6abe94
Open pack map
fetch2002.5 KiB209 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=209
git.response.bytes=2565
git.response.cache=MISS
git.response.duration_ms=209
git.fetch.wants=150606a1e7bbce839392ce3e74d651e9a4077529
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2565
git.pkt.payload.max=2538
git.sideband.band1.bytes=2538
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e2e458b9f72947
Request
9b667fb0…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format