master
1#!/usr/bin/env perl
2#***************************************************************************
3# _ _ ____ _
4# Project ___| | | | _ \| |
5# / __| | | | |_) | |
6# | (__| |_| | _ <| |___
7# \___|\___/|_| \_\_____|
8#
9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
10#
11# This software is licensed as described in the file COPYING, which
12# you should have received as part of this distribution. The terms
13# are also available at https://curl.se/docs/copyright.html.
14#
15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
16# copies of the Software, and permit persons to whom the Software is
17# furnished to do so, under the terms of the COPYING file.
18#
19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
20# KIND, either express or implied.
21#
22# SPDX-License-Identifier: curl
23#
24###########################################################################
26use strict;
27use warnings;
29my %whitelist = (
30 'https://curl.se' => 1,
31 'https://curl.se/' => 1,
32 'https://curl.se/bug/' => 1,
33 'https://curl.se/bug/view.cgi' => 1,
34 'https://curl.se/changes.html' => 1,
35 'https://curl.se/dev/advisory.html' => 1,
36 'https://curl.se/dev/builds.html' => 1,
37 'https://curl.se/dev/code-style.html' => 1,
38 'https://curl.se/dev/contribute.html' => 1,
39 'https://curl.se/dev/internals.html' => 1,
40 'https://curl.se/dev/secprocess.html' => 1,
41 'https://curl.se/dev/sourceactivity.html' => 1,
42 'https://curl.se/docs/' => 1,
43 'https://curl.se/docs/caextract.html' => 1,
44 'https://curl.se/docs/copyright.html' => 1,
45 'https://curl.se/docs/http-cookies.html' => 1,
46 'https://curl.se/docs/install.html' => 1,
47 'https://curl.se/docs/knownbugs.html' => 1,
48 'https://curl.se/docs/manpage.html' => 1,
49 'https://curl.se/docs/releases.html' => 1,
50 'https://curl.se/docs/security.html' => 1,
51 'https://curl.se/docs/ssl-ciphers.html' => 1,
52 'https://curl.se/docs/ssl-compared.html' => 1,
53 'https://curl.se/docs/sslcerts.html' => 1,
54 'https://curl.se/docs/thanks.html' => 1,
55 'https://curl.se/docs/todo.html' => 1,
56 'https://curl.se/docs/vulnerabilities.html' => 1,
57 'https://curl.se/download.html' => 1,
58 'https://curl.se/libcurl/' => 1,
59 'https://curl.se/libcurl/c/CURLOPT_SSL_CIPHER_LIST.html' => 1,
60 'https://curl.se/libcurl/c/CURLOPT_SSLVERSION.html' => 1,
61 'https://curl.se/libcurl/c/CURLOPT_TLS13_CIPHERS.html' => 1,
62 'https://curl.se/libcurl/c/libcurl.html' => 1,
63 'https://curl.se/libcurl/c/threadsafe.html' => 1,
64 'https://curl.se/logo/curl-logo.svg' => 1,
65 'https://curl.se/mail/' => 1,
66 'https://curl.se/mail/etiquette.html' => 1,
67 'https://curl.se/mail/list.cgi?list=curl-distros' => 1,
68 'https://curl.se/mail/list.cgi?list=curl-library' => 1,
69 'https://curl.se/rfc/cookie_spec.html' => 1,
70 'https://curl.se/rfc/rfc2255.txt' => 1,
71 'https://curl.se/sponsors.html' => 1,
72 'https://curl.se/support.html' => 1,
73 'https://curl.se/windows/' => 1,
75 'https://testclutch.curl.se/' => 1,
77 'https://github.com/curl/curl-fuzzer' => 1,
78 'https://github.com/curl/curl-www' => 1,
79 'https://github.com/curl/curl/wcurl' => 1,
81 );
83my %url;
84my %flink;
86my $dry;
87if(defined $ARGV[0] && $ARGV[0] eq "--dry-run") {
88 $dry = 1;
89 shift @ARGV;
90}
92# list all files to scan for links
93my @files = qx(git ls-files appveyor.yml docs include lib scripts src);
95sub storelink {
96 my ($f, $line, $link) = @_;
97 my $o = $link;
99 if($link =~ /^\#/) {
100 # ignore local-only links
101 return;
102 }
103 # cut off any anchor
104 $link =~ s:\#.*\z::;
106 if($link =~ /^(https|http):/) {
107 if($whitelist{$link}) {
108 #print "-- whitelisted: $link\n";
109 $whitelist{$link}++;
110 }
111 # example.com is used as example
112 elsif($link =~ /^https:\/\/(.*)example.(com|org|net)/) {
113 #print "-- example: $link\n";
114 }
115 # so is using the .example TLD
116 elsif($link =~ /^https:\/\/(.*)\.example(\/|$|:)/) {
117 #print "-- .example: $link\n";
118 }
119 # so is using anything on localhost
120 elsif($link =~ /^http(s|):\/\/localhost/) {
121 #print "-- localhost: $link\n";
122 }
123 # ignore all links to curl's github repo
124 elsif($link =~ /^https:\/\/github.com\/curl\/curl(\/|$)/) {
125 #print "-- curl github repo: $link\n";
126 }
127 elsif($link =~ /^(https|http):\/\/[0-9.]+(\/|$)/) {
128 #print "-- IPv4 number: $link\n";
129 }
130 else {
131 #print "ADD '$link'\n";
132 $url{$link} .= "$f:$line ";
133 }
134 return;
135 }
137 # a file link
138 my $dir = $f;
139 $dir =~ s:([^/]*\z)::;
141 if($link =~ s/(^\/)//) {
142 # link starts with a slash, now removed
143 $dir = "";
144 }
145 else {
146 while($link =~ s:^\.\.\/::) {
147 $dir =~ s:([^/]*)\/\z::;
148 }
149 }
151 $flink{"./$dir$link"} .= "$f:$line ";
152}
154sub findlinks {
155 my ($f) = @_;
156 my $line = 1;
157 open(F, "<:crlf", $f) or
158 return;
160 # is it a markdown extension?
161 my $md = ($f =~ /\.md$/i);
163 while(<F>) {
164 chomp;
165 if($md && /\]\(([^)]*)/) {
166 my $link = $1;
167 #print "$f:$line $link\n";
168 storelink($f, $line, $link);
169 }
170 # ignore trailing: dot, double quote, single quote, asterisk, hash,
171 # comma, question mark, colon, closing parenthesis, backslash,
172 # closing angle bracket, whitespace, pipe, backtick, semicolon
173 elsif(/(https:\/\/[a-z0-9.\/:%_+@-]+[^."'*\#,?:\)> \t|`;\\])/i) {
174 #print "RAW '$_'\n";
175 storelink($f, $line, $1);
176 }
177 $line++;
178 }
179 close(F);
180}
182sub checkurl {
183 my ($url) = @_;
185 if($whitelist{$url}) {
186 #print STDERR "$url is whitelisted\n";
187 return 0;
188 }
190 $url =~ s/\+/%2B/g;
191 my @content;
192 if(open(my $fh, '-|', 'curl', '-ILfsm10', '--retry', '2', '--retry-delay', '5', '--retry-max-time', '60',
193 '-A', 'Mozilla/curl.se link-probe', $url)) {
194 @content = <$fh>;
195 close $fh;
196 }
197 if(!$content[0]) {
198 print "FAIL: $url\n";
199 return 1; # fail
200 }
201 print "OK: $url\n";
202 return 0; # ok
203}
205for my $f (@files) {
206 chomp $f;
207 if($f !~ /\/mdlinkcheck$/) {
208 findlinks($f);
209 }
210}
212for my $u (sort keys %whitelist) {
213 if($whitelist{$u} == 1) {
214 printf STDERR "warning: unused whitelist entry: '$u'\n";
215 }
216}
218if($dry) {
219 for my $u (sort keys %url) {
220 print "$u\n";
221 }
222 exit;
223}
225my $error;
226my @errlist;
227for my $u (sort keys %url) {
228 my $r = checkurl($u);
230 if($r) {
231 for my $f (split(/ /, $url{$u})) {
232 push @errlist, sprintf "%s ERROR links to missing URL %s\n", $f, $u;
233 $error++;
234 }
235 }
236}
238for my $l (sort keys %flink) {
239 if(! -r $l) {
240 for my $f (split(/ /, $flink{$l})) {
241 push @errlist, sprintf "%s ERROR links to missing file %s\n", $f, $l;
242 $error++;
243 }
244 }
245}
247printf "Checked %d URLs\n", scalar(keys %url);
248if($error) {
249 print "$error URLs had problems:\n";
250 for(@errlist) {
251 print $_;
252 }
253}
254exit 1 if($error);
Diagnostics4 git calls · 335 ms
CallStatusSizeTime0335 ms
discover200152 B5 ms
git.upload-pack discover http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=3 git.response.bytes=152 git.server.agent=gitty/1.0 git.server.object_format=sha1 git.server.caps=agent ls-refs fetch object-format git.server.fetch=shallow filter sideband-all git.protocol.version=2 git.pkt.frames=8 git.pkt.data=6 git.pkt.flush=2 git.pkt.delim=0 git.pkt.bytes=152 git.pkt.payload.max=34
ls-refs200149 B3 ms
git.upload-pack ls-refs http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=ls-refs git.upstream.dur_ms=3 git.response.bytes=149 git.response.cache=UPDATING git.response.duration_ms=3 git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=149 git.pkt.payload.max=78 git.ls_refs.count=2 git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit27 ms
git.upload-pack fetch http.response.status_code=200 git.upstream.cf_ray=- git.upstream.phase=fetch git.upstream.dur_ms=303 git.response.bytes=293470 git.response.cache=HIT git.response.duration_ms=27 git.fetch.filter=blob:none git.fetch.filter.requested=tree:0 git.fetch.filter.fallback=blob:none git.fetch.filter.learned=false git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d git.fetch.haves.count=0 git.fetch.deepen=10 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=10 git.pkt.data=8 git.pkt.flush=1 git.pkt.delim=1 git.pkt.bytes=293470 git.pkt.payload.max=65516 git.sideband.band1.bytes=293354 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0 git.pack.objects=87 git.pack.bytes=293354 git.pack.ofs_delta=0 git.pack.ref_delta=1 git.pack.chain.max=1 git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0 git.pack.full=86Open pack map
fetch2002.5 KiB303 ms
git.upload-pack fetch http.response.status_code=200 git.response.bytes=2571 git.response.cache=MISS git.response.duration_ms=303 git.fetch.wants=c3a8861d385180a18a2f569a93473a6e970db929 git.fetch.haves.count=0 git.fetch.deepen=0 git.fetch.thin_pack=false git.fetch.ofs_delta=true git.protocol.version=2 git.pkt.frames=3 git.pkt.data=2 git.pkt.flush=1 git.pkt.delim=0 git.pkt.bytes=2571 git.pkt.payload.max=2550 git.sideband.band1.bytes=2549 git.sideband.band2.bytes=0 git.sideband.band3.bytes=0
Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.
- Remote
- https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
- Ref
- refs/heads/master
- Commit
- 31c911efe5652725e930f4d2e60c9060e5f6646d
- cf-ray
- a5e1dd1bae0fecfa
- Request
- ad904ec8…
- Protocol
- v2 · sha1
- Agent
- gitty/1.0
- Filter
- advertised
- Fetch
- shallow filter sideband-all
- Capabilities
- agent ls-refs fetch object-format