Artifactspublic-v0

curl

master
ply4 git calls38 ms292 KiB2 cachedprotocol v2
379 lines13 KiBexecutableRaw
1#!/bin/sh
3# wcurl - a simple wrapper around curl to easily download files.
4#
5# Requires curl >= 7.46.0 (2015)
6#
7# Copyright (C) Samuel Henrique <samueloph@debian.org>, Sergio Durigan
8# Junior <sergiodj@debian.org> and many contributors, see the AUTHORS
9# file.
11# Permission to use, copy, modify, and distribute this software for any purpose
12# with or without fee is hereby granted, provided that the above copyright
13# notice and this permission notice appear in all copies.
15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
18# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
19# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
21# OR OTHER DEALINGS IN THE SOFTWARE.
23# Except as contained in this notice, the name of a copyright holder shall not be
24# used in advertising or otherwise to promote the sale, use or other dealings in
25# this Software without prior written authorization of the copyright holder.
27# SPDX-License-Identifier: curl
29# Stop on errors, usage of unset variables and disable glob expansion.
30set -euf
32VERSION="2026.10.12"
34PROGRAM_NAME="$(basename "$0")"
35readonly PROGRAM_NAME
37# Display the version.
38print_version()
40 cat << _EOF_
41${VERSION}
42_EOF_
45# Display the program usage.
46usage()
48 cat << _EOF_
49${PROGRAM_NAME} -- a simple wrapper around curl to easily download files.
51Usage: ${PROGRAM_NAME} <URL>...
52 ${PROGRAM_NAME} [--curl-options <CURL_OPTIONS>]... [--no-decode-filename] [-o|-O|--output <PATH>] [--dry-run] [--] <URL>...
53 ${PROGRAM_NAME} [--curl-options=<CURL_OPTIONS>]... [--no-decode-filename] [--output=<PATH>] [--dry-run] [--] <URL>...
54 ${PROGRAM_NAME} -h|--help
55 ${PROGRAM_NAME} -V|--version
57Options:
59 --curl-options <CURL_OPTIONS>: Specify extra options to be passed when invoking curl. May be
60 specified more than once.
62 -o, -O, --output <PATH>: Use the provided output path instead of getting it from the URL. If
63 multiple URLs are provided, resulting files share the same name with a
64 number appended to the end (curl >= 7.83.0). If this option is provided
65 multiple times, only the last value is considered.
67 --no-decode-filename: Do not percent-decode the output filename, even if the percent-encoding in
68 the URL was done by wcurl, e.g.: The URL contained whitespace.
70 --dry-run: Do not actually execute curl, print what would be invoked.
72 -V, --version: Print version information.
74 -h, --help: Print this usage message.
76 <CURL_OPTIONS>: Any option supported by curl can be set here. This is not used by wcurl; it is
77 instead forwarded to the curl invocation. The value is split on whitespace and
78 quotes are not interpreted; use '--config <FILE>' for arguments containing
79 whitespace. Never build it from untrusted input.
81 <URL>: URL to be downloaded. Anything that is not a parameter is considered
82 an URL. Whitespace is percent-encoded and the URL is passed to curl, which
83 then performs the parsing. May be specified more than once.
84_EOF_
87# Display an error message and bail out.
88error()
90 printf "%s\n" "$*" >&2
91 exit 1
94# Extra curl options provided by the user.
95# This is set per-URL for every URL provided.
96# Some options are global, but we are erroring on the side of needlessly setting
97# them multiple times instead of causing issues with parameters that needs to
98# be set per-URL.
99CURL_OPTIONS=""
101# The URLs to be downloaded.
102URLS=""
104# Variable used to be set to the percent-decoded filename parsed from the URL, unless
105# --output or --no-decode-filename are used.
106OUTPUT_PATH=""
107HAS_USER_SET_OUTPUT="false"
109# The parameters that are passed per-URL to curl.
110readonly PER_URL_PARAMETERS="\
111 --fail \
112 --globoff \
113 --location \
114 --proto-default https \
115 --remote-time \
116 --retry 5 "
118# Valid percent-encode codes that are considered unsafe to be decoded.
119# This is a list of space-separated percent-encoded uppercase
120# characters.
121# 2F = /
122# 5C = \
123# 3A = :
124# 7F = DEL
125readonly UNSAFE_PERCENT_ENCODE="%2F %5C %3A %7F"
127# Whether to invoke curl or not.
128DRY_RUN="false"
130# Sanitize parameters.
131sanitize()
133 if [ -z "${URLS}" ]; then
134 error "You must provide at least one URL to download."
135 fi
137 # Split the extra curl options while IFS still has its default value, then
138 # join them back with spaces, so that restricting IFS below does not change
139 # the way the user's options are split.
140 # shellcheck disable=SC2086
141 set -- ${CURL_OPTIONS}
143 # Split the list of URLs on spaces only. Spaces are percent-encoded before a
144 # URL is added to the list, so every URL is a single field, and a tab or a
145 # newline inside one URL cannot turn it into more than one curl transfer.
146 IFS=" "
147 CURL_OPTIONS="$*"
149 readonly CURL_OPTIONS URLS DRY_RUN HAS_USER_SET_OUTPUT
152# Indicate via exit code whether the string given in the first parameter
153# consists solely of characters from the string given in the second parameter.
154# In other words, it returns 0 if the first parameter only contains characters
155# from the second parameter, e.g.: Are $1 characters a subset of $2 characters?
156is_subset_of()
158 case "${1}" in
159 *[!${2}]* | '') return 1 ;;
160 esac
163# Indicate via exit code whether the HTML code given in the first
164# parameter is safe to be decoded.
165is_safe_percent_encode()
167 upper_str=$(printf "%s" "${1}" | tr "[:lower:]" "[:upper:]")
168 for unsafe in ${UNSAFE_PERCENT_ENCODE}; do
169 if [ "${unsafe}" = "${upper_str}" ]; then
170 return 1
171 fi
172 done
174 return 0
177# Print the given string percent-decoded.
178percent_decode()
180 # Encodings of control characters (00-1F) are passed through without decoding.
181 # Iterate on the input character-by-character, decoding it.
182 printf "%s\n" "${1}" | fold -w1 | while IFS= read -r decode_out; do
183 # If character is a "%", read the next character as decode_hex1.
184 if [ "${decode_out}" = % ] && IFS= read -r decode_hex1; then
185 decode_out="${decode_out}${decode_hex1}"
186 # If there is one more character, read it as decode_hex2.
187 if IFS= read -r decode_hex2; then
188 decode_out="${decode_out}${decode_hex2}"
189 # Skip decoding if this is a control character (00-1F).
190 # Skip decoding if DECODE_FILENAME is not "true".
191 if [ "${DECODE_FILENAME}" = "true" ] \
192 && is_subset_of "${decode_hex1}" "23456789abcdefABCDEF" \
193 && is_subset_of "${decode_hex2}" "0123456789abcdefABCDEF" \
194 && is_safe_percent_encode "${decode_out}"; then
195 # Use printf to decode it into octal and then decode it to the final format.
196 decode_out="$(printf "%b" "\\$(printf %o "0x${decode_hex1}${decode_hex2}")")"
197 fi
198 fi
199 fi
200 printf %s "${decode_out}"
201 done
204# Print the percent-decoded filename portion of the given URL.
205get_url_filename()
207 # Remove protocol, query string and URL fragments if present.
208 hostname_and_path="$(printf %s "${1}" | sed -e 's,^[^/]*//,,' -e 's,[?#].*$,,')"
209 # If what remains contains a slash, there is a path; return it percent-decoded.
210 case "${hostname_and_path}" in
211 # sed to remove everything preceding the last '/', e.g.: "example/something" becomes "something"
212 # sed to also replace '\' with the percent_encoded %5C and ':' with %3A
213 */*)
214 filename="$(percent_decode "$(printf %s "${hostname_and_path}" | sed -e 's,^.*/,,' -e 's,\\,%5C,g' -e 's,:,%3A,g')")"
215 [ "${filename}" = "-" ] && filename=""
216 printf %s "${filename}"
217 ;;
218 esac
219 # No slash means there was only a hostname and no path; return empty string.
222# Execute curl with the list of URLs provided by the user.
223exec_curl()
225 CMD="curl "
227 # Store version to check if it supports --no-clobber, --parallel and --parallel-max-host.
228 curl_version=$($CMD --version | cut -f2 -d' ' | head -n1)
229 curl_version_major=$(echo "$curl_version" | cut -f1 -d.)
230 curl_version_minor=$(echo "$curl_version" | cut -f2 -d.)
232 CURL_NO_CLOBBER=""
233 CURL_PARALLEL=""
235 if [ "${curl_version_major}" -ge 8 ]; then
236 CURL_NO_CLOBBER="--no-clobber"
237 CURL_PARALLEL="--parallel --parallel-max-host 5"
239 # --parallel-max-host is only supported since 8.16.0.
240 if [ "${curl_version_major}" -eq 8 ] && [ "${curl_version_minor}" -lt 16 ]; then
241 CURL_PARALLEL="--parallel"
242 fi
243 elif [ "${curl_version_major}" -eq 7 ]; then
244 # --no-clobber is only supported since 7.83.0.
245 if [ "${curl_version_minor}" -ge 83 ]; then
246 CURL_NO_CLOBBER="--no-clobber"
247 fi
248 # --parallel is only supported since 7.66.0.
249 if [ "${curl_version_minor}" -ge 66 ]; then
250 CURL_PARALLEL="--parallel"
251 fi
252 fi
254 # Detecting whether we need --parallel. It is easier to rely on
255 # the shell's argument parsing.
256 # shellcheck disable=SC2086
257 set -- $URLS
259 # If there are less than two URLs, do not set the parallel flag.
260 if [ "$#" -lt 2 ]; then
261 CURL_PARALLEL=""
262 elif [ "${HAS_USER_SET_OUTPUT}" = "true" ] && [ -z "${CURL_NO_CLOBBER}" ]; then
263 error "Using '--output' with multiple URLs requires curl >= 7.83.0 ('--no-clobber')."
264 fi
266 # Start assembling the command.
267 #
268 # We use 'set --' here (again) because (a) we do not have arrays on
269 # POSIX shell, and (b) we need better control over the way we
270 # split arguments.
271 #
272 # shellcheck disable=SC2086
273 set -- ${CMD} ${CURL_PARALLEL}
275 NEXT_PARAMETER=""
276 for url in ${URLS}; do
277 # If the user did not provide an output path, define one.
278 if [ "${HAS_USER_SET_OUTPUT}" = "false" ]; then
279 OUTPUT_PATH="$(get_url_filename "${url}")"
280 # If we could not get a path from the URL, use the default: index.html.
281 [ -z "${OUTPUT_PATH}" ] && OUTPUT_PATH=index.html
282 fi
283 # shellcheck disable=SC2086
284 set -- "$@" ${NEXT_PARAMETER} ${PER_URL_PARAMETERS} ${CURL_NO_CLOBBER} --output "${OUTPUT_PATH}" ${CURL_OPTIONS} --url "${url}"
285 NEXT_PARAMETER="--next"
286 done
288 if [ "${DRY_RUN}" = "false" ]; then
289 exec "$@"
290 else
291 printf "%s\n" "$@"
292 fi
295# Default to decoding the output filename
296DECODE_FILENAME="true"
298# Use "${1-}" in order to avoid errors because of 'set -u'.
299while [ -n "${1-}" ]; do
300 case "${1}" in
301 --curl-options=*)
302 opt=$(printf "%s\n" "${1}" | sed 's/^--curl-options=//')
303 CURL_OPTIONS="${CURL_OPTIONS} ${opt}"
304 ;;
306 --curl-options)
307 if [ -z "${2-}" ]; then
308 error "Option '${1}' requires an argument."
309 fi
310 shift
311 CURL_OPTIONS="${CURL_OPTIONS} ${1}"
312 ;;
314 --dry-run)
315 DRY_RUN="true"
316 ;;
318 --output=*)
319 opt=$(printf "%s\n" "${1}" | sed 's/^--output=//')
320 HAS_USER_SET_OUTPUT="true"
321 OUTPUT_PATH="${opt}"
322 ;;
324 -o | -O | --output)
325 if [ -z "${2-}" ]; then
326 error "Option '${1}' requires an argument."
327 fi
328 shift
329 HAS_USER_SET_OUTPUT="true"
330 OUTPUT_PATH="${1}"
331 ;;
333 -o* | -O*)
334 opt=$(printf "%s\n" "${1}" | sed 's/^-[oO]//')
335 HAS_USER_SET_OUTPUT="true"
336 OUTPUT_PATH="${opt}"
337 ;;
339 --no-decode-filename)
340 DECODE_FILENAME="false"
341 ;;
343 -h | --help)
344 usage
345 exit 0
346 ;;
348 -V | --version)
349 print_version
350 exit 0
351 ;;
353 --)
354 # This is the start of the list of URLs.
355 shift
356 for url in "$@"; do
357 # Encode whitespace into %20, since wget supports those URLs.
358 newurl=$(printf "%s\n" "${url}" | sed 's/ /%20/g')
359 URLS="${URLS} ${newurl}"
360 done
361 break
362 ;;
364 -*)
365 error "Unknown option: '$1'."
366 ;;
368 *)
369 # This must be a URL.
370 # Encode whitespace into %20, since wget supports those URLs.
371 newurl=$(printf "%s\n" "${1}" | sed 's/ /%20/g')
372 URLS="${URLS} ${newurl}"
373 ;;
374 esac
375 shift
376done
378sanitize
379exec_curl
Diagnostics4 git calls · 38 ms
CallStatusSizeTime038 ms
discover200152 B3 ms
git.upload-pack discover
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=152
git.server.agent=gitty/1.0
git.server.object_format=sha1
git.server.caps=agent ls-refs fetch object-format
git.server.fetch=shallow filter sideband-all
git.protocol.version=2
git.pkt.frames=8
git.pkt.data=6
git.pkt.flush=2
git.pkt.delim=0
git.pkt.bytes=152
git.pkt.payload.max=34
ls-refs200149 B3 ms
git.upload-pack ls-refs
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=ls-refs
git.upstream.dur_ms=3
git.response.bytes=149
git.response.cache=UPDATING
git.response.duration_ms=3
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=149
git.pkt.payload.max=78
git.ls_refs.count=2
git.ls_refs.head=refs/heads/master
fetchblob:none · deepen 10200287 KiB · hit3 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=3
git.response.bytes=293470
git.response.cache=HIT
git.response.duration_ms=3
git.fetch.filter=blob:none
git.fetch.filter.requested=tree:0
git.fetch.filter.fallback=blob:none
git.fetch.filter.learned=false
git.fetch.wants=31c911efe5652725e930f4d2e60c9060e5f6646d
git.fetch.haves.count=0
git.fetch.deepen=10
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=10
git.pkt.data=8
git.pkt.flush=1
git.pkt.delim=1
git.pkt.bytes=293470
git.pkt.payload.max=65516
git.sideband.band1.bytes=293354
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0
git.pack.objects=87
git.pack.bytes=293354
git.pack.ofs_delta=0
git.pack.ref_delta=1
git.pack.chain.max=1
git.pack.trailer=6a646bcda6740bab42e321b51998b52ea15146c0
git.pack.full=86
Open pack map
fetch2004.8 KiB · hit32 ms
git.upload-pack fetch
http.response.status_code=200
git.upstream.cf_ray=-
git.upstream.phase=fetch
git.upstream.dur_ms=31
git.response.bytes=4922
git.response.cache=HIT
git.response.duration_ms=32
git.fetch.wants=d4914ee5aa3735ccbc624a5d9bb059924e616a8c
git.fetch.haves.count=0
git.fetch.deepen=0
git.fetch.thin_pack=false
git.fetch.ofs_delta=true
git.protocol.version=2
git.pkt.frames=3
git.pkt.data=2
git.pkt.flush=1
git.pkt.delim=0
git.pkt.bytes=4922
git.pkt.payload.max=4901
git.sideband.band1.bytes=4900
git.sideband.band2.bytes=0
git.sideband.band3.bytes=0

Each row is one HTTP round trip ply made to the git server for this page, on one timeline. discover asks what the server speaks, ls-refs lists branch tips, fetch brings objects by id. A hit was served by Workers Cache. Tap a row for its wire details.

Remote
https://5ad2236b39de4c8bd8f6270ea63f3ea9.artifacts.cloudflare.net/git/public-v0/curl.git
Ref
refs/heads/master
Commit
31c911efe5652725e930f4d2e60c9060e5f6646d
cf-ray
a5e1a88bc9e92c38
Request
4cddbeb3…
Protocol
v2 · sha1
Agent
gitty/1.0
Filter
advertised
Fetch
shallow filter sideband-all
Capabilities
agent ls-refs fetch object-format