GitLabgitlab-org

cli

main
ply8 git calls322 ms99 KiB6 cachedprotocol v2
149 lines3.8 KiBRaw
1package artifact
3import (
4 "archive/zip"
5 "bytes"
6 "fmt"
7 "io"
8 "os"
9 "path/filepath"
10 "strings"
12 gitlab "gitlab.com/gitlab-org/api/client-go/v3"
14 "gitlab.com/gitlab-org/cli/internal/config"
15 "gitlab.com/gitlab-org/cli/internal/dbg"
16 "gitlab.com/gitlab-org/cli/internal/glrepo"
17 "gitlab.com/gitlab-org/cli/internal/utils"
20const (
21 // Read limit is 4GB
22 defaultZIPReadLimit int64 = 4 * 1024 * 1024 * 1024
23 defaultZIPFileLimit int = 100000
26func ensurePathIsCreated(filename string) error {
27 dir, _ := filepath.Split(filename)
29 if _, err := os.Stat(filename); os.IsNotExist(err) {
30 err = os.MkdirAll(dir, 0o700) // Create your file
31 if err != nil {
32 return fmt.Errorf("could not create new path: %w", err)
33 }
34 }
35 return nil
38func readZip(artifact *bytes.Reader, path string, listPaths bool, zipReadLimit int64, zipFileLimit int, out io.Writer) error {
39 zipReader, err := zip.NewReader(artifact, artifact.Size())
40 if err != nil {
41 return err
42 }
44 if !config.CheckPathExists(path) {
45 if err := os.Mkdir(path, 0o755); err != nil {
46 return err
47 }
48 }
50 if !strings.HasSuffix(path, "/") {
51 path += "/"
52 }
54 var written int64 = 0
55 if len(zipReader.File) > zipFileLimit {
56 return fmt.Errorf("zip archive includes too many files: limit is %d files", zipFileLimit)
57 }
59 for _, v := range zipReader.File {
60 sanitizedAssetName := utils.SanitizePathName(v.Name)
62 destDir, err := filepath.Abs(path)
63 if err != nil {
64 return fmt.Errorf("resolving absolute download directory path: %w", err)
65 }
66 destPath := filepath.Join(destDir, sanitizedAssetName)
67 if !strings.HasPrefix(destPath, destDir) {
68 return fmt.Errorf("invalid file path name")
69 }
71 dbg.Debug("Writing:", destPath)
73 if v.FileInfo().IsDir() {
74 if err := os.MkdirAll(destPath, v.Mode()); err != nil {
75 return err
76 }
77 continue
78 }
80 writtenPerFile, err := extractZipEntry(zipReader, v, destPath, zipReadLimit, listPaths, out)
81 if err != nil {
82 return err
83 }
85 written += writtenPerFile
86 if written >= zipReadLimit {
87 return fmt.Errorf("extracted zip too large: limit is %d bytes", zipReadLimit)
88 }
89 }
90 return nil
93// extractZipEntry is split out of readZip's loop so that both file handles are
94// released per entry. Deferring them inside the loop held every handle open
95// until the whole archive finished, up to the file-count limit.
96func extractZipEntry(zipReader *zip.Reader, v *zip.File, destPath string, zipReadLimit int64, listPaths bool, out io.Writer) (int64, error) {
97 srcFile, err := zipReader.Open(v.Name)
98 if err != nil {
99 return 0, err
100 }
101 defer srcFile.Close()
103 if err := ensurePathIsCreated(destPath); err != nil {
104 return 0, err
105 }
107 symlinkCheck, _ := os.Lstat(destPath)
108 if symlinkCheck != nil && symlinkCheck.Mode()&os.ModeSymlink != 0 {
109 return 0, fmt.Errorf("can't extract: a file in the artifact would overwrite a symbolic link")
110 }
112 dstFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, v.Mode())
113 if err != nil {
114 return 0, err
115 }
116 defer dstFile.Close()
118 written, err := io.Copy(dstFile, io.LimitReader(srcFile, zipReadLimit))
119 if err != nil {
120 return 0, err
121 }
123 if listPaths {
124 fmt.Fprintln(out, friendlyPath(destPath)) //nolint:forbidigo // out is a generic io.Writer; production caller passes os.Stdout directly, not IOStreams
125 }
127 return written, nil
130func friendlyPath(path string) string {
131 cwd, err := os.Getwd()
132 if err != nil {
133 return path
134 }
135 rel, err := filepath.Rel(cwd, path)
136 if err != nil {
137 return path
138 }
139 return rel
142func DownloadArtifacts(apiClient *gitlab.Client, repo glrepo.Interface, path string, listPaths bool, refName string, jobName string) error {
143 artifact, _, err := apiClient.Jobs.DownloadArtifactsFile(repo.FullName(), refName, &gitlab.DownloadArtifactsFileOptions{Job: &jobName}, nil)
144 if err != nil {
145 return err
146 }
148 return readZip(artifact, path, listPaths, defaultZIPReadLimit, defaultZIPFileLimit, os.Stdout)
Diagnostics8 git calls · 322 ms
CallStatusSizeTime0322 ms
discover200185 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit4 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=4
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchtree:02001.8 KiB · hit1 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=1
git.response.bytes=1857
git.response.cache=HIT
git.response.duration_ms=1
git.fetch.filter=tree:0
git.fetch.wants=dce174ff8c632a19d17eea99c8a9b7c37fb2b592
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1857
git.pkt.payload.max=1830
git.sideband.band1.bytes=1830
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1830
git.pack.trailer=fb03183b7909876778f740d196d7516811003517
Open pack map
fetchtree:0200169 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=169
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=014f8e3d040dd5bbfbad786ad7dada84dc7810ce
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=169
git.pkt.payload.max=142
git.sideband.band1.bytes=142
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=142
git.pack.trailer=47a57f7b9b54feb1a2cb60673fb63b93f55a03c8
Open pack map
fetchtree:0200169 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=169
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=d94ab6b39f4e57708ce09cd3b84c4cc080d0cc18
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=169
git.pkt.payload.max=142
git.sideband.band1.bytes=142
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=142
git.pack.trailer=4bd95bfd76a3b40ee067e0aaec3ca757f3f5feb5
Open pack map
fetch2001.5 KiB309 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=309
git.response.bytes=1564
git.response.cache=MISS
git.response.duration_ms=309
git.fetch.wants=61977491bda80c9f59e09f98086a78eb0f346112
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1564
git.pkt.payload.max=1537
git.sideband.band1.bytes=1537
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e2fdd0ef04cf81
Request
04c1d80c…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format