GitLabgitlab-org

cli

main
ply7 git calls260 ms99 KiB5 cachedprotocol v2
204 lines7.6 KiBRaw
1package proxy
3import (
4 "net/http"
5 "regexp"
6 "slices"
7 "strings"
9 "gitlab.com/gitlab-org/cli/internal/dependencyfirewall/policy"
12// MavenMatcher recognizes Maven artifact downloads and uploads against a
13// public Maven upstream (repo.maven.apache.org and mirrors). The path shape
14// is ".../<group/path>/<artifact>/<version>/<artifact>-<version>.<ext>",
15// where the group's dots are path segments. Enforcement targets the primary
16// artifacts listed in mavenPrimaryExts (.jar, .pom, .aar, .war); checksum and
17// signature sidecars pass through.
18type MavenMatcher struct {
19 // Mirrors, when it knows them, are the mirror URLs Maven downloads
20 // through; without them every prefix a mirror could add is tried.
21 Mirrors *MavenMirrors
24// mavenPrimaryExts are the artifact extensions that trigger a policy check.
25// Sidecars (.sha1, .md5, .sha256, .sha512, .asc) are intentionally omitted.
26var mavenPrimaryExts = []string{".jar", ".pom", ".aar", ".war"}
28func (mm MavenMatcher) Match(req *http.Request) Match {
29 if req == nil {
30 return Match{}
31 }
32 op := policy.Download
33 if req.Method == http.MethodPut || req.Method == http.MethodPost {
34 op = policy.Upload
35 }
36 if m, ok := mm.Mirrors.match(req, op); ok {
37 return m
38 }
39 name, version, result, central := mavenCoordinate(req.URL.Path)
40 switch result {
41 case mavenNotArtifact:
42 return Match{}
43 case mavenUnparseable:
44 return unparseableMatch("maven", op, "unparseable maven artifact coordinate")
45 }
46 // The exact Central layout has no prefix to recover, and readings that drop
47 // segments would only collide with relocated groups like dom4j:dom4j.
48 var alternates []policy.Coordinate
49 if !central {
50 alternates = mavenGroupAlternates(name, version)
51 }
52 return Match{
53 Matched: true,
54 Pass: true,
55 Coordinate: policy.Coordinate{Ecosystem: "maven", Name: name, Version: version},
56 Operation: op,
57 Alternates: alternates,
58 }
61// mavenMaxGroupDepth caps mavenGroupAlternates so a pathologically deep path
62// can't generate unbounded policy calls: at most twice this many readings.
63const mavenMaxGroupDepth = 10
65// mavenGroupAlternates builds the group readings a mirror's path prefix could
66// produce, shortest first, since a rule's real target is far more likely short.
67func mavenGroupAlternates(name, version string) []policy.Coordinate {
68 group, artifact, ok := strings.Cut(name, ":")
69 if !ok {
70 return nil
71 }
72 parts := strings.Split(group, ".")
73 if len(parts) < 2 {
74 return nil
75 }
76 var alts []policy.Coordinate
77 for start := len(parts) - 1; start >= 1; start-- {
78 // The publisher chooses the groupId's depth but a repository prefix is
79 // short, so keep short readings and those that drop few leading segments.
80 if len(parts)-start > mavenMaxGroupDepth && start > mavenMaxGroupDepth {
81 continue
82 }
83 seg := parts[start:]
84 // A doubled slash leaves an empty segment; that's not a real group.
85 if slices.Contains(seg, "") {
86 continue
87 }
88 alts = append(alts, policy.Coordinate{
89 Ecosystem: "maven",
90 Name: strings.Join(seg, ".") + ":" + artifact,
91 Version: version,
92 })
93 }
94 return alts
97// mavenCoordinateResult classifies a request path relative to the Maven
98// artifact shape: not a Maven artifact fetch, a filename claiming a coordinate
99// that does not match its path (fail closed), or a parsed coordinate.
100type mavenCoordinateResult int
102const (
103 mavenNotArtifact mavenCoordinateResult = iota
104 mavenUnparseable
105 mavenParsed
108// mavenCoordinate also reports whether path is the exact Central layout under
109// a leading maven2/, whose coordinate needs no alternates.
110func mavenCoordinate(path string) (string, string, mavenCoordinateResult, bool) {
111 // path is req.URL.Path, which Go has already percent-decoded once. Decoding
112 // again would let a double-encoded request (e.g. %2520) present the matcher
113 // a different path than the upstream Maven server sees, and because an
114 // unmatched request bypasses the firewall that is a policy-bypass vector.
115 // Match against the once-decoded path the origin actually receives.
116 rest := strings.Trim(path, "/")
117 // Treating maven2/ as Central is a guess: a mirror served under
118 // maven2/<repo>/ is read as <repo>.<group>:<artifact> with no alternates.
119 // A mirror named in settings.xml is matched exactly instead.
120 central := strings.HasPrefix(rest, "maven2/")
121 name, version, result := mavenLayoutCoordinate(strings.TrimPrefix(rest, "maven2/"))
122 return name, version, result, central && result == mavenParsed
125// mavenLayoutCoordinate parses rest, a path relative to a repository's root,
126// as the Maven repository layout.
127func mavenLayoutCoordinate(rest string) (string, string, mavenCoordinateResult) {
128 if rest == "" {
129 return "", "", mavenNotArtifact
130 }
131 segs := strings.Split(rest, "/")
132 // Sidecar and metadata requests (checksums, signatures,
133 // maven-metadata.xml) are not artifact fetches and pass through untouched.
134 file := segs[len(segs)-1]
135 if !mavenIsPrimary(file) {
136 return "", "", mavenNotArtifact
137 }
138 // Every request in a gradle or mvn session reaches this matcher, so a
139 // primary extension alone does not make it repository traffic: GitHub
140 // release jars, Ivy layouts and tool downloads carry no coordinate and are
141 // forwarded like any other download.
142 if len(segs) < 4 {
143 return "", "", mavenNotArtifact
144 }
145 version := segs[len(segs)-2]
146 artifact := segs[len(segs)-3]
147 group := strings.Join(segs[:len(segs)-3], ".")
148 if !mavenFileMatchesCoordinate(file, artifact, version) {
149 // A filename that names the directory's artifact but not its version
150 // (foo-10.jar under foo/1/) claims a coordinate we cannot trust: fail
151 // closed.
152 if strings.HasPrefix(file, artifact+"-") {
153 return "", "", mavenUnparseable
154 }
155 return "", "", mavenNotArtifact
156 }
157 return group + ":" + artifact, version, mavenParsed
160// snapshotSuffix is the version-directory marker for a Maven snapshot.
161const snapshotSuffix = "-SNAPSHOT"
163// mavenSnapshotTimestamp matches the "<yyyyMMdd>.<HHmmss>-<buildnumber>"
164// segment Maven substitutes for "SNAPSHOT" in a deployed snapshot artifact's
165// filename, for example "20240101.123456-1" in
166// "slf4j-api-1.0-20240101.123456-1.jar".
167var mavenSnapshotTimestamp = regexp.MustCompile(`^\d{8}\.\d{6}-\d+`)
169// mavenFileMatchesCoordinate reports whether file names the artifact at
170// version. The direct case requires "<artifact>-<version>" as the filename
171// prefix followed by a boundary: "." for the extension (slf4j-api-1.0.jar) or
172// "-" for a classifier (slf4j-api-1.0-sources.jar). The boundary matters
173// because one version can prefix another ("1" prefixes "10"): without it
174// "foo-10.jar" would match version "1" and report the wrong coordinate, and
175// since unmatched means allow, mask the version actually being fetched.
176// Snapshot deployments are the exception: under a "<base>-SNAPSHOT" version
177// directory, Maven names each uploaded artifact with a unique timestamp
178// instead of the literal "SNAPSHOT" (for example
179// "slf4j-api-1.0-20240101.123456-1.jar" under "1.0-SNAPSHOT/"), so the direct
180// prefix check would miss it and let every timestamped snapshot download
181// bypass policy. Accept that timestamped form too so snapshots are evaluated.
182func mavenFileMatchesCoordinate(file, artifact, version string) bool {
183 if rest, ok := strings.CutPrefix(file, artifact+"-"+version); ok {
184 if strings.HasPrefix(rest, ".") || strings.HasPrefix(rest, "-") {
185 return true
186 }
187 }
188 if base, ok := strings.CutSuffix(version, snapshotSuffix); ok {
189 rest, matched := strings.CutPrefix(file, artifact+"-"+base+"-")
190 if matched && mavenSnapshotTimestamp.MatchString(rest) {
191 return true
192 }
193 }
194 return false
197func mavenIsPrimary(file string) bool {
198 for _, ext := range mavenPrimaryExts {
199 if strings.HasSuffix(file, ext) {
200 return true
201 }
202 }
203 return false
Diagnostics7 git calls · 260 ms
CallStatusSizeTime0260 ms
discover200185 B2 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=185
git.server.agent=git/2.56.g1891de4-Linux
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch server-option object-format
git.server.fetch=shallow wait-for-done filter
git.protocol.version=2
git.pkt.frames=9
git.pkt.data=7
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=185
git.pkt.payload.max=35
ls-refs20058 KiB · hit2 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=2
git.response.bytes=59849
git.response.cache=HIT
git.response.duration_ms=2
git.protocol.version=2
git.pkt.frames=731
git.pkt.data=730
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=59849
git.pkt.payload.max=143
git.ls_refs.count=730
git.ls_refs.head=refs/heads/main
fetchtree:0 · deepen 1020032 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=4
git.response.bytes=32926
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=22
git.pkt.data=20
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=32926
git.pkt.payload.max=32046
git.sideband.band1.bytes=32046
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=80
git.pack.bytes=32046
git.pack.ofs_delta=36
git.pack.ref_delta=0
git.pack.chain.max=2
git.pack.trailer=cb0bad5d7ffc514895d8b21c4dd470327f90bc91
git.pack.full=44
Open pack map
fetchcombine:blob:none+tree:2 · deepen 12004.2 KiB · hit4 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=4273
git.response.cache=HIT
git.response.duration_ms=4
git.fetch.filter=combine:blob:none+tree:2
git.fetch.wants=0dd7574dce9e385c5c30540e6c15657adc6bdce9
git.fetch.haves.count=0
git.fetch.deepen=1
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=7
git.pkt.data=5
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=4273
git.pkt.payload.max=4173
git.sideband.band1.bytes=4173
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=11
git.pack.bytes=4173
git.fetch.wide.objects=11
git.fetch.wide.extra_bytes=3930
git.fetch.wide.blobs=0
fetchtree:0200361 B · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=2
git.response.bytes=361
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=tree:0
git.fetch.wants=506cba622ceda1473ee3cde63969809c144b7247
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=361
git.pkt.payload.max=334
git.sideband.band1.bytes=334
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=334
git.pack.trailer=26d32585a3a474717e00a318584ccd16918c9d32
Open pack map
fetchtree:02001.0 KiB · hit2 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=249
git.response.bytes=1051
git.response.cache=HIT
git.response.duration_ms=2
git.fetch.filter=tree:0
git.fetch.wants=7a2a90605315ba5b37f63c08726e6b2caa26e48d
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=1051
git.pkt.payload.max=1024
git.sideband.band1.bytes=1024
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=1
git.pack.bytes=1024
git.pack.trailer=7503ef36a75d3595b473035c69e639fd0ce41598
Open pack map
fetch2003.0 KiB249 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=3117
git.response.cache=MISS
git.response.duration_ms=249
git.fetch.wants=ed5233500b522e0776050e2df8ec50754f016e00
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=4
git.pkt.data=3
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=3117
git.pkt.payload.max=3090
git.sideband.band1.bytes=3090
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://gitlab.com/gitlab-org/cli
Ref
refs/heads/main
Commit
0dd7574dce9e385c5c30540e6c15657adc6bdce9
cf-ray
a5e22040faefa87d
Request
8fbb7d4c…
Protocol
v2 · sha1
Agent
git/2.56.g1891de4-Linux
Filter
advertised
Fetch
shallow wait-for-done filter
Capabilities
agent ls-refs fetch server-option object-format