Artifactspublic-v0

curl

master
ply4 git calls335 ms289 KiB1 cachedprotocol v2
254 lines7.2 KiBexecutableRaw
1#!/usr/bin/env perl
2#***************************************************************************
3# _ _ ____ _
4# Project ___| | | | _ \| |
5# / __| | | | |_) | |
6# | (__| |_| | _ <| |___
7# \___|\___/|_| \_\_____|
8#
9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
11# This software is licensed as described in the file COPYING, which
12# you should have received as part of this distribution. The terms
13# are also available at https://curl.se/docs/copyright.html.
15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
16# copies of the Software, and permit persons to whom the Software is
17# furnished to do so, under the terms of the COPYING file.
19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20# KIND, either express or implied.
22# SPDX-License-Identifier: curl
24###########################################################################
26use strict;
27use warnings;
29my %whitelist = (
30 'https://curl.se' => 1,
31 'https://curl.se/' => 1,
32 'https://curl.se/bug/' => 1,
33 'https://curl.se/bug/view.cgi' => 1,
34 'https://curl.se/changes.html' => 1,
35 'https://curl.se/dev/advisory.html' => 1,
36 'https://curl.se/dev/builds.html' => 1,
37 'https://curl.se/dev/code-style.html' => 1,
38 'https://curl.se/dev/contribute.html' => 1,
39 'https://curl.se/dev/internals.html' => 1,
40 'https://curl.se/dev/secprocess.html' => 1,
41 'https://curl.se/dev/sourceactivity.html' => 1,
42 'https://curl.se/docs/' => 1,
43 'https://curl.se/docs/caextract.html' => 1,
44 'https://curl.se/docs/copyright.html' => 1,
45 'https://curl.se/docs/http-cookies.html' => 1,
46 'https://curl.se/docs/install.html' => 1,
47 'https://curl.se/docs/knownbugs.html' => 1,
48 'https://curl.se/docs/manpage.html' => 1,
49 'https://curl.se/docs/releases.html' => 1,
50 'https://curl.se/docs/security.html' => 1,
51 'https://curl.se/docs/ssl-ciphers.html' => 1,
52 'https://curl.se/docs/ssl-compared.html' => 1,
53 'https://curl.se/docs/sslcerts.html' => 1,
54 'https://curl.se/docs/thanks.html' => 1,
55 'https://curl.se/docs/todo.html' => 1,
56 'https://curl.se/docs/vulnerabilities.html' => 1,
57 'https://curl.se/download.html' => 1,
58 'https://curl.se/libcurl/' => 1,
59 'https://curl.se/libcurl/c/CURLOPT_SSL_CIPHER_LIST.html' => 1,
60 'https://curl.se/libcurl/c/CURLOPT_SSLVERSION.html' => 1,
61 'https://curl.se/libcurl/c/CURLOPT_TLS13_CIPHERS.html' => 1,
62 'https://curl.se/libcurl/c/libcurl.html' => 1,
63 'https://curl.se/libcurl/c/threadsafe.html' => 1,
64 'https://curl.se/logo/curl-logo.svg' => 1,
65 'https://curl.se/mail/' => 1,
66 'https://curl.se/mail/etiquette.html' => 1,
67 'https://curl.se/mail/list.cgi?list=curl-distros' => 1,
68 'https://curl.se/mail/list.cgi?list=curl-library' => 1,
69 'https://curl.se/rfc/cookie_spec.html' => 1,
70 'https://curl.se/rfc/rfc2255.txt' => 1,
71 'https://curl.se/sponsors.html' => 1,
72 'https://curl.se/support.html' => 1,
73 'https://curl.se/windows/' => 1,
75 'https://testclutch.curl.se/' => 1,
77 'https://github.com/curl/curl-fuzzer' => 1,
78 'https://github.com/curl/curl-www' => 1,
79 'https://github.com/curl/curl/wcurl' => 1,
81 );
83my %url;
84my %flink;
86my $dry;
87if(defined $ARGV[0] && $ARGV[0] eq "--dry-run") {
88 $dry = 1;
89 shift @ARGV;
92# list all files to scan for links
93my @files = qx(git ls-files appveyor.yml docs include lib scripts src);
95sub storelink {
96 my ($f, $line, $link) = @_;
97 my $o = $link;
99 if($link =~ /^\#/) {
100 # ignore local-only links
101 return;
102 }
103 # cut off any anchor
104 $link =~ s:\#.*\z::;
106 if($link =~ /^(https|http):/) {
107 if($whitelist{$link}) {
108 #print "-- whitelisted: $link\n";
109 $whitelist{$link}++;
110 }
111 # example.com is used as example
112 elsif($link =~ /^https:\/\/(.*)example.(com|org|net)/) {
113 #print "-- example: $link\n";
114 }
115 # so is using the .example TLD
116 elsif($link =~ /^https:\/\/(.*)\.example(\/|$|:)/) {
117 #print "-- .example: $link\n";
118 }
119 # so is using anything on localhost
120 elsif($link =~ /^http(s|):\/\/localhost/) {
121 #print "-- localhost: $link\n";
122 }
123 # ignore all links to curl's github repo
124 elsif($link =~ /^https:\/\/github.com\/curl\/curl(\/|$)/) {
125 #print "-- curl github repo: $link\n";
126 }
127 elsif($link =~ /^(https|http):\/\/[0-9.]+(\/|$)/) {
128 #print "-- IPv4 number: $link\n";
129 }
130 else {
131 #print "ADD '$link'\n";
132 $url{$link} .= "$f:$line ";
133 }
134 return;
135 }
137 # a file link
138 my $dir = $f;
139 $dir =~ s:([^/]*\z)::;
141 if($link =~ s/(^\/)//) {
142 # link starts with a slash, now removed
143 $dir = "";
144 }
145 else {
146 while($link =~ s:^\.\.\/::) {
147 $dir =~ s:([^/]*)\/\z::;
148 }
149 }
151 $flink{"./$dir$link"} .= "$f:$line ";
154sub findlinks {
155 my ($f) = @_;
156 my $line = 1;
157 open(F, "<:crlf", $f) or
158 return;
160 # is it a markdown extension?
161 my $md = ($f =~ /\.md$/i);
163 while(<F>) {
164 chomp;
165 if($md && /\]\(([^)]*)/) {
166 my $link = $1;
167 #print "$f:$line $link\n";
168 storelink($f, $line, $link);
169 }
170 # ignore trailing: dot, double quote, single quote, asterisk, hash,
171 # comma, question mark, colon, closing parenthesis, backslash,
172 # closing angle bracket, whitespace, pipe, backtick, semicolon
173 elsif(/(https:\/\/[a-z0-9.\/:%_+@-]+[^."'*\#,?:\)> \t|`;\\])/i) {
174 #print "RAW '$_'\n";
175 storelink($f, $line, $1);
176 }
177 $line++;
178 }
179 close(F);
182sub checkurl {
183 my ($url) = @_;
185 if($whitelist{$url}) {
186 #print STDERR "$url is whitelisted\n";
187 return 0;
188 }
190 $url =~ s/\+/%2B/g;
191 my @content;
192 if(open(my $fh, '-|', 'curl', '-ILfsm10', '--retry', '2', '--retry-delay', '5', '--retry-max-time', '60',
193 '-A', 'Mozilla/curl.se link-probe', $url)) {
194 @content = <$fh>;
195 close $fh;
196 }
197 if(!$content[0]) {
198 print "FAIL: $url\n";
199 return 1; # fail
200 }
201 print "OK: $url\n";
202 return 0; # ok
205for my $f (@files) {
206 chomp $f;
207 if($f !~ /\/mdlinkcheck$/) {
208 findlinks($f);
209 }
212for my $u (sort keys %whitelist) {
213 if($whitelist{$u} == 1) {
214 printf STDERR "warning: unused whitelist entry: '$u'\n";
215 }
218if($dry) {
219 for my $u (sort keys %url) {
220 print "$u\n";
221 }
222 exit;
225my $error;
226my @errlist;
227for my $u (sort keys %url) {
228 my $r = checkurl($u);
230 if($r) {
231 for my $f (split(/ /, $url{$u})) {
232 push @errlist, sprintf "%s ERROR links to missing URL %s\n", $f, $u;
233 $error++;
234 }
235 }
238for my $l (sort keys %flink) {
239 if(! -r $l) {
240 for my $f (split(/ /, $flink{$l})) {
241 push @errlist, sprintf "%s ERROR links to missing file %s\n", $f, $l;
242 $error++;
243 }
244 }
247printf "Checked %d URLs\n", scalar(keys %url);
248if($error) {
249 print "$error URLs had problems:\n";
250 for(@errlist) {
251 print $_;
252 }
254exit 1 if($error);
Diagnostics4 git calls · 335 ms
CallStatusSizeTime0335 ms
discover200152 B5 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit27 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=303
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=27
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2002.5 KiB303 ms
git.upload-pack fetch
http.response.status_code=200
git.response.bytes=2571
git.response.cache=MISS
git.response.duration_ms=303
git.fetch.wants=c3a8861d385180a18a2f569a93473a6e970db929
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=2571
git.pkt.payload.max=2550
git.sideband.band1.bytes=2549
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e1dd1bae0fecfa
Request
ad904ec8…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format